Testing and refactoring

Get acceptance tests for github repo sync working

Get all GitHub acceptance tests passing

Add syncing count to repo sync

Update sync states, add to views, update syncing to start when creating project repo

Add tests for add_merged in GithubPullRequest

Add more sync tests and some docs

Author: joshsmith

lib/code_corps/github/sync/comment/comment/changeset.ex

@@ -7,7 +7,6 @@ defmodule CodeCorps.GitHub.Sync.Comment.Comment.Changeset do
   alias CodeCorps.{
     Comment,
     GithubComment,
-    ProjectGithubRepo,
     Services.MarkdownRendererService,
     Task,
     User,
@@ -17,8 +16,7 @@ defmodule CodeCorps.GitHub.Sync.Comment.Comment.Changeset do
   alias Ecto.Changeset
 
   @doc ~S"""
-  Constructs a changeset for syncing a task when processing a GitHub Comment
-  payload
+  Constructs a changeset for syncing a task from a GitHub API Comment payload.
   """
   @spec build_changeset(Comment.t, map, GithubComment.t, Task.t, User.t) :: Changeset.t
   def build_changeset(
@@ -34,6 +32,9 @@ defmodule CodeCorps.GitHub.Sync.Comment.Comment.Changeset do
     comment |> update_changeset(attrs)
   end
 
+  @doc ~S"""
+  Constructs a changeset for syncing a task from a `GithubComment` record.
+  """
   @spec build_changeset(Comment.t, GithubComment.t, Task.t, User.t) :: Changeset.t
   def build_changeset(
     %Comment{id: comment_id} = comment,

lib/code_corps/github/sync/sync.ex

@@ -2,7 +2,6 @@ defmodule CodeCorps.GitHub.Sync do
 
   alias CodeCorps.{
     Comment,
-    GithubIssue,
     GitHub,
     GitHub.Utils.ResultAggregator,
     GithubPullRequest,
@@ -108,44 +107,62 @@ defmodule CodeCorps.GitHub.Sync do
   defp sync_step({:ok, _} = result, _step), do: result
   defp sync_step({:error, _ = error}, _step), do: {:error, error}
 
-  @spec mark(GithubRepo.t, String.t) :: {:ok, GithubRepo.t} | {:error, Changeset.t}
-  defp mark(%GithubRepo{} = repo, sync_state) when is_binary(sync_state) do
-    IO.puts("------------#{sync_state}-----------------")
+  @spec mark_repo(GithubRepo.t, String.t, Keyword.t) :: {:ok, GithubRepo.t} | {:error, Changeset.t}
+  defp mark_repo(%GithubRepo{} = repo, sync_state, opts \\ []) do
+    params = build_sync_params(sync_state, opts)
     repo
-    |> GithubRepo.update_sync_changeset(%{sync_state: sync_state})
+    |> GithubRepo.update_sync_changeset(params)
     |> Repo.update
   end
-  @spec mark(ProjectGithubRepo.t, String.t) :: {:ok, GithubRepo.t} | {:error, Changeset.t}
-  defp mark(%ProjectGithubRepo{} = project_github_repo, sync_state) when is_binary(sync_state) do
-    IO.puts("------------#{sync_state}-----------------")
+
+  @spec mark_project_repo(ProjectGithubRepo.t, String.t, Keyword.t) :: {:ok, GithubRepo.t} | {:error, Changeset.t}
+  defp mark_project_repo(%ProjectGithubRepo{} = project_github_repo, sync_state, opts \\ []) do
+    params = build_sync_params(sync_state, opts)
     project_github_repo
-    |> ProjectGithubRepo.update_sync_changeset(%{sync_state: sync_state})
+    |> ProjectGithubRepo.update_sync_changeset(params)
     |> Repo.update
   end
 
+
+  @count_fields [:syncing_comments_count, :syncing_issues_count, :syncing_pull_requests_count]
+
+  defp build_sync_params(sync_state, opts) do
+    Enum.reduce @count_fields, %{sync_state: sync_state}, fn field, acc ->
+      put_sync_opt(opts, field, acc)
+    end
+  end
+
+  defp put_sync_opt(opts, key, map) do
+    case Keyword.get(opts, key) do
+      nil -> map
+      count -> map |> Map.put(key, count)
+    end
+  end
+
   @spec sync_repo(GithubRepo.t) :: {:ok, GithubRepo.t}
   def sync_repo(%GithubRepo{} = repo) do
-    with {:ok, pr_payloads} <- repo |> GitHub.API.Repository.pulls |> sync_step(:fetch_pull_requests),
-         {:ok, repo} <- repo |> mark("syncing_pull_requests"),
+    with {:ok, repo} <- repo |> mark_repo("fetching_pull_requests"),
+         {:ok, pr_payloads} <- repo |> GitHub.API.Repository.pulls |> sync_step(:fetch_pull_requests),
+         {:ok, repo} <- repo |> mark_repo("syncing_github_pull_requests", [syncing_pull_requests_count: pr_payloads |> Enum.count]),
          {:ok, _pull_requests} <- pr_payloads |> Enum.map(&Sync.PullRequest.GithubPullRequest.create_or_update_pull_request(repo, &1)) |> ResultAggregator.aggregate |> sync_step(:sync_pull_requests),
-         {:ok, repo} <- repo |> mark("fetching_issues"),
+         {:ok, repo} <- repo |> mark_repo("fetching_issues"),
          {:ok, issue_payloads} <- repo |> GitHub.API.Repository.issues |> sync_step(:fetch_issues),
-         {:ok, repo} <- repo |> mark("syncing_issues"),
+         {:ok, repo} <- repo |> mark_repo("syncing_github_issues", [syncing_issues_count: issue_payloads |> Enum.count]),
          {:ok, _issues} <- issue_payloads |> Enum.map(&Sync.Issue.GithubIssue.create_or_update_issue(repo, &1)) |> ResultAggregator.aggregate |> sync_step(:sync_issues),
-         {:ok, repo} <- repo |> mark("fetching_comments"),
+         {:ok, repo} <- repo |> mark_repo("fetching_comments"),
          {:ok, comment_payloads} <- repo |> GitHub.API.Repository.issue_comments |> sync_step(:fetch_comments),
-         {:ok, repo} <- repo |> mark("syncing_comments"),
+         {:ok, repo} <- repo |> mark_repo("syncing_github_comments", [syncing_comments_count: comment_payloads |> Enum.count]),
          {:ok, _comments} <- comment_payloads |> Enum.map(&Sync.Comment.GithubComment.create_or_update_comment(repo, &1)) |> ResultAggregator.aggregate |> sync_step(:sync_comments),
-         {:ok, repo} <- repo |> mark("receiving_webhooks")
+         {:ok, repo} <- repo |> mark_repo("receiving_webhooks")
     do
       {:ok, repo}
     else
-      {:error, :fetch_pull_requests} -> repo |> mark("errored_fetching_pull_requests")
-      {:error, :sync_pull_requests} -> repo |> mark("errored_syncing_pull_requests")
-      {:error, :fetch_issues} -> repo |> mark("errored_fetching_issues")
-      {:error, :sync_issues} -> repo |> mark("errored_syncing_issues")
-      {:error, :fetch_comments} -> repo |> mark("errored_fetching_comments")
-      {:error, :sync_comments} -> repo |> mark("errored_syncing_comments")
+      {:error, :fetch_pull_requests} -> repo |> mark_repo("errored_fetching_pull_requests")
+      {:error, :sync_pull_requests} -> repo |> mark_repo("errored_syncing_pull_requests")
+      {:error, :fetch_issues} -> repo |> mark_repo("errored_fetching_issues")
+      {:error, :sync_issues} -> repo |> mark_repo("errored_syncing_issues")
+      {:error, :fetch_comments} -> repo |> mark_repo("errored_fetching_comments")
+      {:error, :sync_comments} -> repo |> mark_repo("errored_syncing_comments")
     end
   end
 
@@ -154,20 +171,20 @@ defmodule CodeCorps.GitHub.Sync do
       project_github_repo
       |> preload_project_github_repo
 
-    with {:ok, project_github_repo} <- project_github_repo |> mark("syncing_repo"),
+    with {:ok, project_github_repo} <- project_github_repo |> mark_project_repo("syncing_github_repo"),
          {:ok, %GithubRepo{sync_state: "receiving_webhooks"}} <- repo |> sync_repo(),
          project_github_repo <- Repo.get(ProjectGithubRepo, project_github_repo.id) |> preload_project_github_repo(),
-         {:ok, project_github_repo} <- project_github_repo |> mark("syncing_tasks"),
+         {:ok, project_github_repo} <- project_github_repo |> mark_project_repo("syncing_tasks"),
          {:ok, _tasks} <- project_github_repo |> Sync.Issue.Task.sync_project_github_repo() |> sync_step(:sync_tasks),
-         {:ok, project_github_repo} <- project_github_repo |> mark("syncing_comments"),
+         {:ok, project_github_repo} <- project_github_repo |> mark_project_repo("syncing_comments"),
          {:ok, _comments} <- project_github_repo |> Sync.Comment.Comment.sync_project_github_repo() |> sync_step(:sync_comments),
-         {:ok, project_github_repo} <- project_github_repo |> mark("synced")
+         {:ok, project_github_repo} <- project_github_repo |> mark_project_repo("synced")
     do
       {:ok, project_github_repo}
     else
-      {:ok, %GithubRepo{}} -> project_github_repo |> mark("errored_syncing_repo")
-      {:error, :sync_tasks} -> repo |> mark("errored_syncing_tasks")
-      {:error, :sync_comments} -> repo |> mark("errored_syncing_comments")
+      {:ok, %GithubRepo{}} -> project_github_repo |> mark_project_repo("errored_syncing_github_repo")
+      {:error, :sync_tasks} -> repo |> mark_project_repo("errored_syncing_tasks")
+      {:error, :sync_comments} -> repo |> mark_project_repo("errored_syncing_comments")
     end
   end
 

lib/code_corps/model/github_repo.ex

@@ -51,11 +51,11 @@ defmodule CodeCorps.GithubRepo do
     ~w{
       unsynced
       fetching_pull_requests errored_fetching_pull_requests
-      syncing_pull_requests errored_syncing_pull_requests
+      syncing_github_pull_requests errored_syncing_github_pull_requests
       fetching_issues errored_fetching_issues
-      syncing_issues errored_syncing_issues
+      syncing_github_issues errored_syncing_github_issues
       fetching_comments errored_fetching_comments
-      syncing_comments errored_syncing_comments
+      syncing_github_comments errored_syncing_github_comments
       receiving_webhooks
     }
   end

lib/code_corps/model/project_github_repo.ex

@@ -41,7 +41,7 @@ defmodule CodeCorps.ProjectGithubRepo do
   def sync_states do
     ~w{
       unsynced
-      syncing_repo errored_syncing_repo
+      syncing_github_repo errored_syncing_github_repo
       syncing_tasks errored_syncing_tasks
       syncing_comments errored_syncing_comments
       synced

lib/code_corps_web/controllers/project_github_repo_controller.ex

@@ -2,6 +2,7 @@ defmodule CodeCorpsWeb.ProjectGithubRepoController do
   @moduledoc false
   use CodeCorpsWeb, :controller
 
+  alias Task.Supervisor, as: TaskSupervisor
   alias CodeCorps.{Analytics.SegmentTracker, ProjectGithubRepo, User, Helpers.Query}
 
   action_fallback CodeCorpsWeb.FallbackController
@@ -28,6 +29,8 @@ defmodule CodeCorpsWeb.ProjectGithubRepoController do
          {:ok, :authorized} <- current_user |> Policy.authorize(:create, %ProjectGithubRepo{}, params),
          {:ok, %ProjectGithubRepo{} = project_github_repo} <- create_project_repo_changeset(params) |> Repo.insert do
 
+      TaskSupervisor.start_child(:background_processor, fn -> CodeCorps.GitHub.Sync.sync_project_github_repo(project_github_repo) end)
+
       current_user |> track_created(project_github_repo)
 
       conn |> put_status(:created) |> render("show.json-api", data: project_github_repo)

lib/code_corps_web/views/github_repo_view.ex

@@ -5,7 +5,8 @@ defmodule CodeCorpsWeb.GithubRepoView do
 
   attributes [:github_account_avatar_url, :github_account_id,
   :github_account_login, :github_account_type, :github_id, :inserted_at,
-  :name, :updated_at]
+  :name, :syncing_comments_count, :syncing_issues_count,
+  :syncing_pull_requests_count, :sync_state, :updated_at]
 
   has_one :github_app_installation, type: "github-app-installation", field: :github_app_installation_id
 end

lib/code_corps_web/views/project_github_repo_view.ex

@@ -3,6 +3,8 @@ defmodule CodeCorpsWeb.ProjectGithubRepoView do
   use CodeCorpsWeb, :view
   use JaSerializer.PhoenixView
 
+  attributes [:sync_state]
+
   has_one :github_repo, type: "github-repo", field: :github_repo_id
   has_one :project, type: "project", field: :project_id
 end