Adding support for handling rate limits errors.

Author: tmoskovitch

README.md

@@ -44,7 +44,7 @@ Add this dependency to your project's POM:
 <dependency>
   <groupId>com.twitter</groupId>
   <artifactId>twitter-api-java-sdk</artifactId>
-  <version>1.1.1</version>
+  <version>1.1.2</version>
 </dependency>
 ```
 
@@ -53,7 +53,7 @@ Add this dependency to your project's POM:
 Add this dependency to your project's build file:
 
 ```groovy
-implementation "com.twitter:twitter-api-java-sdk:1.1.1"
+implementation "com.twitter:twitter-api-java-sdk:1.1.2"
 ```
 
 ### Others
@@ -66,7 +66,7 @@ mvn clean package
 
 Then manually install the following JARs:
 
-- `target/twitter-api-java-sdk-1.1.1.jar`
+- `target/twitter-api-java-sdk-1.1.2.jar`
 - `target/lib/*.jar`
 
 ## Twitter Credentials
@@ -201,6 +201,28 @@ You can implement the callback `ApiClientCallback.onAfterRefreshToken()` in orde
 Check this [example](examples/src/main/java/com/twitter/clientlib/auth/OAuth20RefreshToken.java) of implementing `ApiClientCallback`
 
 
+## Rate limits retry mechanism
+
+
+Everyday many thousands of developers make requests to the Twitter developer platform. To help manage the sheer volume of these requests, limits are placed on the number of requests that can be made. These limits help us provide the reliable and scalable API that our developer community relies on.
+
+Each of our APIs use rate limits in different ways. To learn more about these differences between platforms, please review the specific rate limit pages within our specific API sections.
+
+To check connection limits response will return [three headers](https://developer.twitter.com/en/docs/twitter-api/tweets/filtered-stream/integrate/handling-disconnections#:~:text=Rate%20limits%20and%20usage). This is useful to understand how many times you can use the rule endpoint, and how many reconnections attempts are allowed for the streaming endpoint.
+
+The Java SDK provides APIs with a build-in retry mechanism to handle the rate limits. In case of getting an http error code 429, the API will check the response headers and will wait until the rate limit is reset.
+
+In order to use the retry mechanism call the APIs with an additional parameter `retries` as a first argument, check the following example:
+
+
+```java
+  int retries = 4;
+  streamResult = apiInstance.tweets().sampleStream(retries, null, tweetFields, null, null, null, null, 0);
+
+```
+  
+Read more about Filtered stream and [rate limits](https://developer.twitter.com/en/docs/twitter-api/tweets/filtered-stream/integrate/handling-disconnections)
+
 
 ## Documentation for API Endpoints
 

examples/pom.xml

@@ -16,7 +16,7 @@
         <dependency>
             <groupId>com.twitter</groupId>
             <artifactId>twitter-api-java-sdk</artifactId>
-            <version>1.1.1</version>
+            <version>1.1.2</version>
             <scope>compile</scope>
         </dependency>
     </dependencies>

examples/src/main/java/com/twitter/clientlib/HelloWorldStreaming.java

@@ -75,6 +75,11 @@ public static void main(String[] args) {
 //         BufferedReader reader = new BufferedReader(new InputStreamReader(streamResult));
 //         String line = reader.readLine();
 //         while (line != null) {
+//           if(line.isEmpty()) {
+//             System.err.println("==> " + line.isEmpty());
+//             line = reader.readLine();
+//             continue;
+//            }
 //           System.out.println(json.getGson().fromJson(line, localVarReturnType).toString());
 //           line = reader.readLine();
 //         }

pom.xml

@@ -5,7 +5,7 @@
     <artifactId>twitter-api-java-sdk</artifactId>
     <packaging>jar</packaging>
     <name>twitter-api-java-sdk</name>
-    <version>1.1.1</version>
+    <version>1.1.2</version>
     <url>https://github.com/twitterdev/twitter-api-java-sdk</url>
     <description>Twitter API v2 available endpoints</description>
     <scm>

src/main/java/com/twitter/clientlib/ApiClient.java

@@ -232,7 +232,7 @@ private void init() {
         json = new JSON();
 
         // Set default User-Agent.
-        setUserAgent("twitter-api-java-sdk/1.1.1");
+        setUserAgent("twitter-api-java-sdk/1.1.2");
 
         authentications = new HashMap<String, Authentication>();
     }
@@ -1058,7 +1058,7 @@ public InputStream executeStream(Call call, Type returnType) throws ApiException
         try {
              Response response = call.execute();
              if (!response.isSuccessful()) {
-                 throw new ApiException(response.code(), response.message());
+                 throw new ApiException(response.message(), response.code(), response.headers().toMultimap(), null);
              }
              if (response.body() == null) {
                  return null;

src/main/java/com/twitter/clientlib/api/ApiCommon.java

@@ -24,6 +24,9 @@
 
 import com.github.scribejava.core.model.OAuth2AccessToken;
 
+import java.util.Calendar;
+import java.util.List;
+
 import com.twitter.clientlib.ApiClient;
 import com.twitter.clientlib.ApiException;
 
@@ -45,5 +48,38 @@ protected String[] reduceAuthNames(String[] localVarAuthNames) {
   protected boolean isOAUth2AutoRefreshToken() {
     return localVarApiClient.isOAUth2AutoRefreshToken();
   }
+
+  public boolean handleRateLimit(ApiException e, Integer retries) throws ApiException {
+    boolean retryCall = false;
+    if (e.getCode() == 429 && retries > 0) {
+      long timeToWait = getTimeToWait(e);
+      try {
+        Thread.sleep(timeToWait);
+      } catch (InterruptedException ex) {
+        ex.printStackTrace();
+      }
+      retryCall = true;
+    }
+    return retryCall;
+  }
+
+  long getTimeToWait(ApiException e) {
+    long timeToWait = 1000;
+
+    if (isRateLimitRemaining(e)) {
+      List<String> xRateLimitReset = e.getResponseHeaders().get("x-rate-limit-reset");
+      if (xRateLimitReset != null && xRateLimitReset.get(0) != null) {
+        timeToWait = Long.parseLong(
+            xRateLimitReset.get(0)) * 1000 - Calendar.getInstance().getTimeInMillis();
+      }
+    }
+    return timeToWait;
+  }
+
+  boolean isRateLimitRemaining(ApiException e) {
+    List<String> xRateLimitRemaining = e.getResponseHeaders().get("x-rate-limit-remaining");
+    return xRateLimitRemaining != null && xRateLimitRemaining.get(0) != null
+        && Long.parseLong(xRateLimitRemaining.get(0)) == 0;
+  }
 }
 

src/main/java/com/twitter/clientlib/api/BookmarksApi.java

@@ -194,6 +194,24 @@ public GenericTweetsTimelineResponse getUsersIdBookmarks(String id, Integer maxR
         return localVarResp != null ? localVarResp.getData() : null;
     }
 
+   /**
+    * Calls the API using a retry mechanism to handle rate limits errors.
+    *
+    */
+    public GenericTweetsTimelineResponse getUsersIdBookmarks(Integer retries, String id, Integer maxResults, String paginationToken, Set<String> expansions, Set<String> tweetFields, Set<String> userFields, Set<String> mediaFields, Set<String> placeFields, Set<String> pollFields) throws ApiException {
+        GenericTweetsTimelineResponse localVarResp;
+        try{
+          localVarResp = getUsersIdBookmarks(id, maxResults, paginationToken, expansions, tweetFields, userFields, mediaFields, placeFields, pollFields);
+        } catch (ApiException e) {
+          if(handleRateLimit(e, retries)) {
+            return getUsersIdBookmarks(retries - 1, id, maxResults, paginationToken, expansions, tweetFields, userFields, mediaFields, placeFields, pollFields);
+          } else {
+            throw e;
+          }
+        }
+        return localVarResp;
+    }
+
     /**
      * Bookmarks by User
      * Returns Tweet objects that have been bookmarked by the requesting user
@@ -351,6 +369,24 @@ public BookmarkMutationResponse postUsersIdBookmarks(AddBookmarkRequest addBookm
         return localVarResp != null ? localVarResp.getData() : null;
     }
 
+   /**
+    * Calls the API using a retry mechanism to handle rate limits errors.
+    *
+    */
+    public BookmarkMutationResponse postUsersIdBookmarks(Integer retries, AddBookmarkRequest addBookmarkRequest, String id) throws ApiException {
+        BookmarkMutationResponse localVarResp;
+        try{
+          localVarResp = postUsersIdBookmarks(addBookmarkRequest, id);
+        } catch (ApiException e) {
+          if(handleRateLimit(e, retries)) {
+            return postUsersIdBookmarks(retries - 1, addBookmarkRequest, id);
+          } else {
+            throw e;
+          }
+        }
+        return localVarResp;
+    }
+
     /**
      * Add Tweet to Bookmarks
      * Adds a Tweet (ID in the body) to the requesting user&#39;s (in the path) bookmarks
@@ -495,6 +531,24 @@ public BookmarkMutationResponse usersIdBookmarksDelete(String id, String tweetId
         return localVarResp != null ? localVarResp.getData() : null;
     }
 
+   /**
+    * Calls the API using a retry mechanism to handle rate limits errors.
+    *
+    */
+    public BookmarkMutationResponse usersIdBookmarksDelete(Integer retries, String id, String tweetId) throws ApiException {
+        BookmarkMutationResponse localVarResp;
+        try{
+          localVarResp = usersIdBookmarksDelete(id, tweetId);
+        } catch (ApiException e) {
+          if(handleRateLimit(e, retries)) {
+            return usersIdBookmarksDelete(retries - 1, id, tweetId);
+          } else {
+            throw e;
+          }
+        }
+        return localVarResp;
+    }
+
     /**
      * Remove a bookmarked Tweet
      * Removes a Tweet from the requesting user&#39;s bookmarked Tweets.

src/main/java/com/twitter/clientlib/api/ComplianceApi.java

@@ -146,6 +146,24 @@ public SingleComplianceJobResponse createBatchComplianceJob(CreateBatchComplianc
         return localVarResp != null ? localVarResp.getData() : null;
     }
 
+   /**
+    * Calls the API using a retry mechanism to handle rate limits errors.
+    *
+    */
+    public SingleComplianceJobResponse createBatchComplianceJob(Integer retries, CreateBatchComplianceJobRequest createBatchComplianceJobRequest) throws ApiException {
+        SingleComplianceJobResponse localVarResp;
+        try{
+          localVarResp = createBatchComplianceJob(createBatchComplianceJobRequest);
+        } catch (ApiException e) {
+          if(handleRateLimit(e, retries)) {
+            return createBatchComplianceJob(retries - 1, createBatchComplianceJobRequest);
+          } else {
+            throw e;
+          }
+        }
+        return localVarResp;
+    }
+
     /**
      * Create compliance job
      * Creates a compliance for the given job type
@@ -280,6 +298,24 @@ public SingleComplianceJobResponse getBatchComplianceJob(String id) throws ApiEx
         return localVarResp != null ? localVarResp.getData() : null;
     }
 
+   /**
+    * Calls the API using a retry mechanism to handle rate limits errors.
+    *
+    */
+    public SingleComplianceJobResponse getBatchComplianceJob(Integer retries, String id) throws ApiException {
+        SingleComplianceJobResponse localVarResp;
+        try{
+          localVarResp = getBatchComplianceJob(id);
+        } catch (ApiException e) {
+          if(handleRateLimit(e, retries)) {
+            return getBatchComplianceJob(retries - 1, id);
+          } else {
+            throw e;
+          }
+        }
+        return localVarResp;
+    }
+
     /**
      * Get compliance job
      * Returns a single compliance job by ID
@@ -423,6 +459,24 @@ public MultiComplianceJobResponse listBatchComplianceJobs(ComplianceJobType type
         return localVarResp != null ? localVarResp.getData() : null;
     }
 
+   /**
+    * Calls the API using a retry mechanism to handle rate limits errors.
+    *
+    */
+    public MultiComplianceJobResponse listBatchComplianceJobs(Integer retries, ComplianceJobType type, ComplianceJobStatus status) throws ApiException {
+        MultiComplianceJobResponse localVarResp;
+        try{
+          localVarResp = listBatchComplianceJobs(type, status);
+        } catch (ApiException e) {
+          if(handleRateLimit(e, retries)) {
+            return listBatchComplianceJobs(retries - 1, type, status);
+          } else {
+            throw e;
+          }
+        }
+        return localVarResp;
+    }
+
     /**
      * List compliance jobs
      * Returns recent compliance jobs for a given job type and optional job status

src/main/java/com/twitter/clientlib/api/GeneralApi.java

@@ -130,6 +130,24 @@ public Object getOpenApiSpec() throws ApiException {
         return localVarResp != null ? localVarResp.getData() : null;
     }
 
+   /**
+    * Calls the API using a retry mechanism to handle rate limits errors.
+    *
+    */
+    public Object getOpenApiSpec(Integer retries) throws ApiException {
+        Object localVarResp;
+        try{
+          localVarResp = getOpenApiSpec();
+        } catch (ApiException e) {
+          if(handleRateLimit(e, retries)) {
+            return getOpenApiSpec(retries - 1);
+          } else {
+            throw e;
+          }
+        }
+        return localVarResp;
+    }
+
     /**
      * Returns the open api spec document.
      * Full open api spec in JSON format. (See https://github.com/OAI/OpenAPI-Specification/blob/master/README.md)