diff --git a/EventListener/RateLimitAnnotationListener.php b/EventListener/RateLimitAnnotationListener.php
index 76558dc..f1d17bd 100644
--- a/EventListener/RateLimitAnnotationListener.php
+++ b/EventListener/RateLimitAnnotationListener.php
@@ -9,8 +9,6 @@
 use Noxlogic\RateLimitBundle\Exception\RateLimitExceptionInterface;
 use Noxlogic\RateLimitBundle\LimitProcessorInterface;
 use Noxlogic\RateLimitBundle\Service\RateLimitService;
-use Noxlogic\RateLimitBundle\Util\AnnotationLimitProcessor;
-use Noxlogic\RateLimitBundle\Util\PathLimitProcessor;
 use Symfony\Component\EventDispatcher\EventDispatcherInterface;
 use Symfony\Component\HttpFoundation\Request;
 use Symfony\Component\HttpFoundation\Response;
@@ -31,23 +29,23 @@ class RateLimitAnnotationListener extends BaseListener
     protected $rateLimitService;
 
     /**
-     * @var \Noxlogic\RateLimitBundle\Util\PathLimitProcessor
+     * @var LimitProcessorInterface
      */
-    protected $pathLimitProcessor;
+    protected $limitProcessor;
 
     /**
      * @param EventDispatcherInterface $eventDispatcher
      * @param RateLimitService $rateLimitService
-     * @param PathLimitProcessor $pathLimitProcessor
+     * @param LimitProcessorInterface $limitProcessor
      */
     public function __construct(
         EventDispatcherInterface $eventDispatcher,
         RateLimitService $rateLimitService,
-        PathLimitProcessor $pathLimitProcessor
+        LimitProcessorInterface $limitProcessor
     ) {
         $this->eventDispatcher = $eventDispatcher;
         $this->rateLimitService = $rateLimitService;
-        $this->pathLimitProcessor = $pathLimitProcessor;
+        $this->limitProcessor = $limitProcessor;
     }
 
     /**
@@ -65,15 +63,7 @@ public function onKernelController(FilterControllerEvent $event)
             return;
         }
 
-        // Find the best match
-        $annotations = $event->getRequest()->attributes->get('_x-rate-limit', array());
-
-        $limitProcessor = $this->pathLimitProcessor;
-        if ($annotations) {
-            $limitProcessor = new AnnotationLimitProcessor($annotations, $event->getController());
-        }
-
-        $rateLimit = $limitProcessor->getRateLimit($event->getRequest());
+        $rateLimit = $this->limitProcessor->getRateLimit($event->getRequest());
 
         // Another treatment before applying RateLimit ?
         $checkedRateLimitEvent = new CheckedRateLimitEvent($event->getRequest(), $rateLimit);
@@ -85,7 +75,7 @@ public function onKernelController(FilterControllerEvent $event)
             return;
         }
 
-        $key = $this->getKey($limitProcessor, $rateLimit, $event->getRequest());
+        $key = $this->getKey($rateLimit, $event);
 
         $rateLimitInfo = $this->rateLimitService->getRateLimitInfo($key, $rateLimit);
         if (!$rateLimitInfo) {
@@ -139,7 +129,7 @@ protected function findBestMethodMatch(Request $request, array $annotations)
 
         // Empty array, check the path limits
         if (count($annotations) == 0) {
-            return $this->pathLimitProcessor->getRateLimit($request);
+            return $this->limitProcessor->getRateLimit($request);
         }
 
         $best_match = null;
@@ -160,13 +150,14 @@ protected function findBestMethodMatch(Request $request, array $annotations)
         return $best_match;
     }
 
-    private function getKey(LimitProcessorInterface $limitProcessor, RateLimit $rateLimit, Request $request)
+    private function getKey(RateLimit $rateLimit, FilterControllerEvent $event)
     {
         // Let listeners manipulate the key
+        $request = $event->getRequest();
         $keyEvent = new GenerateKeyEvent($request, '', $rateLimit->getPayload());
 
         $keyEvent->addToKey(join('.', $rateLimit->getMethods()));
-        $keyEvent->addToKey($limitProcessor->getRateLimitAlias($request));
+        $keyEvent->addToKey($this->limitProcessor->getRateLimitAlias($request, $event->getController()));
 
         $this->eventDispatcher->dispatch(RateLimitEvents::GENERATE_KEY, $keyEvent);
 
diff --git a/LimitProcessorInterface.php b/LimitProcessorInterface.php
index 469a045..b0bee64 100644
--- a/LimitProcessorInterface.php
+++ b/LimitProcessorInterface.php
@@ -15,7 +15,8 @@ public function getRateLimit(Request $request);
 
     /**
      * @param Request $request
+     * @param callable $controller
      * @return string
      */
-    public function getRateLimitAlias(Request $request);
+    public function getRateLimitAlias(Request $request, callable $controller);
 }
diff --git a/Resources/config/services.xml b/Resources/config/services.xml
index 78b1601..037a623 100644
--- a/Resources/config/services.xml
+++ b/Resources/config/services.xml
@@ -10,6 +10,7 @@
         <parameter key="noxlogic_rate_limit.rate_limit_service.class">Noxlogic\RateLimitBundle\Service\RateLimitService</parameter>
         <parameter key="noxlogic_rate_limit.oauth_key_generate_listener.class">Noxlogic\RateLimitBundle\EventListener\OauthKeyGenerateListener</parameter>
         <parameter key="noxlogic_rate_limit.path_limit_processor.class">Noxlogic\RateLimitBundle\Util\PathLimitProcessor</parameter>
+        <parameter key="noxlogic_rate_limit.annotation_limit_processor.class">Noxlogic\RateLimitBundle\Util\AnnotationLimitProcessor</parameter>
     </parameters>
 
     <services>
@@ -23,6 +24,12 @@
             </call>
         </service>
 
+        <service alias="noxlogic_rate_limit.processor" id="noxlogic_rate_limit.annotation_limit_processor" public="true"/>
+
+        <service id="noxlogic_rate_limit.annotation_limit_processor" class="%noxlogic_rate_limit.annotation_limit_processor.class%" public="false">
+            <argument id="noxlogic_rate_limit.path_limit_processor" type="service"/>
+        </service>
+
         <service id="noxlogic_rate_limit.path_limit_processor" class="%noxlogic_rate_limit.path_limit_processor.class%">
             <argument>%noxlogic_rate_limit.path_limits%</argument>
         </service>
@@ -30,7 +37,7 @@
         <service id="noxlogic_rate_limit.rate_limit_annotation_listener" class="%noxlogic_rate_limit.rate_limit_annotation_listener.class%">
             <argument type="service" id="event_dispatcher" />
             <argument type="service" id="noxlogic_rate_limit.rate_limit_service" />
-            <argument type="service" id="noxlogic_rate_limit.path_limit_processor" />
+            <argument type="service" id="noxlogic_rate_limit.processor" />
 
             <tag name="kernel.event_listener" event="kernel.controller" method="onKernelController" priority="-10" />
 
diff --git a/Util/AnnotationLimitProcessor.php b/Util/AnnotationLimitProcessor.php
index 1155356..8942348 100644
--- a/Util/AnnotationLimitProcessor.php
+++ b/Util/AnnotationLimitProcessor.php
@@ -9,25 +9,25 @@
 class AnnotationLimitProcessor implements LimitProcessorInterface
 {
     /**
-     * @var array
+     * @var LimitProcessorInterface
      */
-    private $annotations;
+    private $fallback;
 
-    /**
-     * @var callable
-     */
-    private $controller;
-
-    public function __construct(array $annotations, callable $controller)
+    public function __construct(LimitProcessorInterface $fallback)
     {
-        $this->annotations = $annotations;
-        $this->controller = $controller;
+        $this->fallback = $fallback;
     }
 
     public function getRateLimit(Request $request)
     {
+        $annotations = $request->attributes->get('_x-rate-limit', array());
+
+        if (!$annotations) {
+            return $this->fallback->getRateLimit($request);
+        }
+
         $best_match = null;
-        foreach ($this->annotations as $annotation) {
+        foreach ($annotations as $annotation) {
             // cast methods to array, even method holds a string
             $methods = is_array($annotation->getMethods()) ? $annotation->getMethods() : array($annotation->getMethods());
 
@@ -44,14 +44,18 @@ public function getRateLimit(Request $request)
         return $best_match;
     }
 
-    public function getRateLimitAlias(Request $request)
+    public function getRateLimitAlias(Request $request, callable $controller)
     {
+        $annotations = $request->attributes->get('_x-rate-limit', array());
+
+        if (!$annotations) {
+            return $this->fallback->getRateLimitAlias($request, $controller);
+        }
+
         if (($route = $request->attributes->get('_route'))) {
             return $route;
         }
 
-        $controller = $this->controller;
-
         if (is_string($controller) && false !== strpos($controller, '::')) {
             $controller = explode('::', $controller);
         }
diff --git a/Util/PathLimitProcessor.php b/Util/PathLimitProcessor.php
index e77545e..6f5d9bc 100644
--- a/Util/PathLimitProcessor.php
+++ b/Util/PathLimitProcessor.php
@@ -56,7 +56,7 @@ public function getMatchedPath(Request $request)
         return $this->getMatchedLimitPath($request);
     }
 
-    public function getRateLimitAlias(Request $request)
+    public function getRateLimitAlias(Request $request, callable $controller)
     {
         return str_replace('/', '.', $this->getMatchedLimitPath($request));
     }