Attempt to implement pagination - initial commit

Author: begedin

lib/code_corps/github/api/api.ex

@@ -14,6 +14,9 @@ defmodule CodeCorps.GitHub.API do
     |> marshall_response()
   end
 
+  defdelegate eager_get_all(url, headers, opts), to: CodeCorps.GitHub.EagerAPI
+  defdelegate lazy_get_all(url, headers, opts), to: CodeCorps.GitHub.StreamAPI
+
   @doc """
   Get access token headers for a given `CodeCorps.User` and
   `CodeCorps.GithubAppInstallation`.

lib/code_corps/github/api/eager_api.ex

@@ -0,0 +1,92 @@
+defmodule CodeCorps.GitHub.EagerAPI do
+  @moduledoc """
+  This module attempts to implement eager loading of a resource, by trying to fetch
+  all of it's pages in paralel.
+
+  This should technically be faster than lazy loading. However, it fails due to
+  timeout errors, even when loading just two pages.
+
+  The assumption is that hackney needs to be configured to allow multiple
+  requests.
+  """
+
+  def eager_get_all(url, headers, options) do
+    {:ok, _status, headers, body} =
+      :hackney.request(:get, url, headers, "", options)
+
+    first_page = Poison.decode!(body)
+
+    case headers |> retrieve_total_pages do
+      1 -> first_page
+      total -> first_page ++ get_remaining_pages(total, url, headers, options)
+    end
+  end
+
+  defp get_remaining_pages(total, url, headers, options) do
+    uri_struct = url |> URI.parse
+    base_query =
+      uri_struct
+      |> Map.get(:query)
+      |> URI.decode_query
+      |> Map.put_new("per_page", 100)
+
+    page_links =
+      2..total
+      |> Enum.to_list
+      |> Enum.map(fn page ->
+        uri_struct |> Map.put(:query, base_query |> Map.put("page", page) |> URI.encode_query)
+        |> URI.to_string
+      end)
+
+    page_links
+    |> Enum.map(&fetch_page(&1, headers, options))
+    |> List.flatten
+  end
+
+  defp fetch_page(url, headers, options) do
+    {:ok, _status, headers, body} =
+      :hackney.request(:get, url, headers, "", options)
+
+    Poison.decode!(body)
+  end
+
+  def retrieve_total_pages(headers) do
+    case headers |> IO.inspect |> List.keyfind("Link", 0, nil) do
+      nil -> 1
+      {"Link", value} -> value |> extract_total_pages
+    end
+  end
+
+  def extract_total_pages(links_string) do
+    links = String.split(links_string, ", ")
+    last_page_url = Enum.map(links, fn link ->
+      [_, name] = Regex.run(~r{rel="([a-z]+)"}, link)
+      [_, url] = Regex.run(~r{<([^>]+)>}, link)
+
+      {name, url}
+    end) |> Enum.into(%{}) |> Map.get("last")
+
+    last_page =
+      last_page_url
+      |> URI.parse
+      |> Map.get(:query)
+      |> URI.decode_query
+      |> Map.get("page")
+      |> String.to_integer
+      |> IO.inspect
+  end
+
+  defp process_page({nil, nil}, _, _) do
+    {:halt, nil}
+  end
+
+  defp process_page({nil, next_page_url}, headers, options) do
+    next_page_url
+    |> fetch_page(headers, options)
+    |> process_page(headers, options)
+  end
+
+  defp process_page({items, next_page_url}, _, _) do
+    {items, {nil, next_page_url}}
+  end
+end

lib/code_corps/github/api/repository.ex

@@ -0,0 +1,54 @@
+defmodule CodeCorps.GitHub.API.Repository do
+  @moduledoc ~S"""
+  Functions for working with issues on GitHub.
+  """
+
+  alias CodeCorps.{
+    GitHub,
+    GitHub.API,
+    GithubAppInstallation,
+    GithubRepo,
+    Task,
+    User
+  }
+
+  @spec issues(GithubRepo.t) :: {:ok, list(map)} | {:error, GitHub.api_error_struct}
+  def issues(%GithubRepo{github_app_installation: %GithubAppInstallation{} = installation} = github_repo) do
+    with {:ok, access_token} <- installation |> API.Installation.get_access_token(),
+      issues <-github_repo |> fetch_issues(access_token) do
+      {:ok, issues}
+    else
+      {:error, error} -> {:error, error}
+    end
+  end
+
+  defp fetch_issues(%GithubRepo{github_account_login: owner, name: repo}, access_token) do
+    # stream/lazy
+
+    before_operation = Timex.now
+
+    results =
+      "repos/#{owner}/#{repo}/issues?per_page=8"
+      |> GitHub.lazy_get_all(%{}, [access_token: access_token])
+      |> Enum.to_list
+      |> List.flatten
+
+    after_operation = Timex.now
+
+    IO.puts("Retrieving #{results |> Enum.count} took #{Timex.diff(after_operation, before_operation)} microseconds")
+
+    # eager
+
+    before_operation = Timex.now
+
+    results =
+      "repos/#{owner}/#{repo}/issues?per_page=8"
+      |> GitHub.eager_get_all(%{}, [access_token: access_token])
+      |> Enum.to_list
+      |> List.flatten
+
+    after_operation = Timex.now
+
+    IO.puts("Retrieving #{results |> Enum.count} took #{Timex.diff(after_operation, before_operation)} microseconds")
+  end
+end

lib/code_corps/github/api/stream_api.ex

@@ -0,0 +1,60 @@
+defmodule CodeCorps.GitHub.StreamAPI do
+  @moduledoc """
+  This module implements retrieval of all pages for a resource in a lazy fashion,
+  through `Stream.resource`.
+
+  That means we load one page at a time and simply concatentate the results.
+  """
+
+  def lazy_get_all(url, headers, options) do
+    Stream.resource(
+      fn -> fetch_page(url, headers, options) end,
+      &process_page(&1, headers, options),
+      fn _ -> nil end
+    )
+  end
+
+  defp fetch_page(url, headers, options) do
+    {:ok, status, headers, body} =
+      :hackney.request(:get, url, headers, "", options)
+
+    items = Poison.decode!(body)
+
+    links_map =
+      headers
+      |> List.keyfind("Link", 0, {nil, nil})
+      |> elem(1)
+      |> parse_links
+    {items, links_map["next"]}
+  end
+
+  def parse_links(nil) do
+    %{}
+  end
+
+  def parse_links(links_string) do
+    links_string
+    links = String.split(links_string, ", ")
+
+    Enum.map(links, fn link ->
+      [_, name] = Regex.run(~r{rel="([a-z]+)"}, link)
+      [_, url] = Regex.run(~r{<([^>]+)>}, link)
+
+      {name, url}
+    end) |> Enum.into(%{})
+  end
+
+  defp process_page({nil, nil}, _, _) do
+    {:halt, nil}
+  end
+
+  defp process_page({nil, next_page_url}, headers, options) do
+    next_page_url
+    |> fetch_page(headers, options)
+    |> process_page(headers, options)
+  end
+
+  defp process_page({items, next_page_url}, _, _) do
+    {items, {nil, next_page_url}}
+  end
+end

lib/code_corps/github/github.ex

@@ -124,6 +124,22 @@ defmodule CodeCorps.GitHub do
     end
   end
 
+  def lazy_get_all(endpoint, headers, options) do
+    api().lazy_get_all(
+      api_url_for(endpoint),
+      headers |> Headers.user_request(options),
+      options |> add_default_options()
+    )
+  end
+
+  def eager_get_all(endpoint, headers, options) do
+    api().eager_get_all(
+      api_url_for(endpoint),
+      headers |> Headers.user_request(options),
+      options |> add_default_options()
+    )
+  end
+
   @token_url "https://github.com/login/oauth/access_token"
 
   @doc """