Merge pull request #248 from LKedward/incremental2

Refactor backend for incremental rebuilds

Author: LKedward

fpm/src/fpm.f90

@@ -8,10 +8,11 @@ module fpm
 use fpm_model, only: fpm_model_t, srcfile_t, build_target_t, &
                     FPM_SCOPE_UNKNOWN, FPM_SCOPE_LIB, &
                     FPM_SCOPE_DEP, FPM_SCOPE_APP, FPM_SCOPE_TEST, &
-                    FPM_TARGET_EXECUTABLE
+                    FPM_TARGET_EXECUTABLE, FPM_TARGET_ARCHIVE
 
 use fpm_sources, only: add_executable_sources, add_sources_from_dir
-use fpm_targets, only: targets_from_sources, resolve_module_dependencies
+use fpm_targets, only: targets_from_sources, resolve_module_dependencies, &
+                        resolve_target_linking
 use fpm_manifest, only : get_package_data, package_config_t
 use fpm_error, only : error_t, fatal_error
 use fpm_manifest_test, only : test_config_t
@@ -240,8 +241,14 @@ subroutine build_model(model, settings, package, error)
         model%link_flags = model%link_flags // " -l" // model%link_libraries(i)%s
     end do
 
+    if (model%targets(1)%ptr%target_type == FPM_TARGET_ARCHIVE) then
+        model%library_file = model%targets(1)%ptr%output_file
+    end if
+
     call resolve_module_dependencies(model%targets,error)
 
+    call resolve_target_linking(model%targets)
+
 end subroutine build_model
 
 
@@ -403,13 +410,7 @@ subroutine cmd_run(settings,test)
 
     end if
 
-    ! NB. To be replaced after incremental rebuild is implemented
-    if (.not.settings%list .and. &
-         any([(.not.exists(executables(i)%s),i=1,size(executables))])) then
-
-        call build_package(model)
-
-    end if
+    call build_package(model)
 
     do i=1,size(executables)
         if (settings%list) then

fpm/src/fpm_backend.f90

@@ -1,113 +1,202 @@
+!> Implements the native fpm build backend
 module fpm_backend
 
-! Implements the native fpm build backend
-
-use fpm_environment, only: run, get_os_type, OS_WINDOWS
-use fpm_filesystem, only: basename, dirname, join_path, exists, mkdir
-use fpm_model, only: fpm_model_t, srcfile_t, build_target_t, FPM_UNIT_MODULE, &
-                     FPM_UNIT_SUBMODULE, FPM_UNIT_SUBPROGRAM, &
-                     FPM_UNIT_CSOURCE, FPM_UNIT_PROGRAM, &
-                     FPM_SCOPE_TEST, FPM_TARGET_OBJECT, FPM_TARGET_ARCHIVE, FPM_TARGET_EXECUTABLE
+use fpm_environment, only: run
+use fpm_filesystem, only: dirname, join_path, exists, mkdir
+use fpm_model, only: fpm_model_t, build_target_t, build_target_ptr, &
+                     FPM_TARGET_OBJECT, FPM_TARGET_ARCHIVE, FPM_TARGET_EXECUTABLE
 
-use fpm_strings, only: split
+use fpm_strings, only: string_cat
 
 implicit none
 
 private
-public :: build_package
+public :: build_package, sort_target, schedule_targets
 
 contains
 
-
+!> Top-level routine to build package described by `model`
 subroutine build_package(model)
     type(fpm_model_t), intent(inout) :: model
 
-    integer :: i, ilib
-    character(:), allocatable :: base, linking, subdir, link_flags
+    integer :: i, j
+    type(build_target_ptr), allocatable :: queue(:)
+    integer, allocatable :: schedule_ptr(:)
 
-    if (.not.exists(model%output_directory)) then
-        call mkdir(model%output_directory)
-    end if
+    ! Need to make output directory for include (mod) files
     if (.not.exists(join_path(model%output_directory,model%package_name))) then
         call mkdir(join_path(model%output_directory,model%package_name))
     end if
 
-    if (model%targets(1)%ptr%target_type == FPM_TARGET_ARCHIVE) then
-        linking = " "//model%targets(1)%ptr%output_file
-    else
-        linking = " "
-    end if
-
-    linking = linking//" "//model%link_flags
-
+    ! Perform depth-first topological sort of targets
     do i=1,size(model%targets)
 
-        call build_target(model,model%targets(i)%ptr,linking)
+        call sort_target(model%targets(i)%ptr)
 
     end do
 
-end subroutine build_package
+    ! Construct build schedule queue
+    call schedule_targets(queue, schedule_ptr, model%targets)
 
+    ! Loop over parallel schedule regions
+    do i=1,size(schedule_ptr)-1
 
+        ! Build targets in schedule region i
+        !$omp parallel do default(shared)
+        do j=schedule_ptr(i),(schedule_ptr(i+1)-1)
+
+            call build_target(model,queue(j)%ptr)
+
+        end do
+
+    end do
+    
+end subroutine build_package
 
-recursive subroutine build_target(model,target,linking)
-    ! Compile Fortran source, called recursively on it dependents
-    !  
-    type(fpm_model_t), intent(in) :: model
-    type(build_target_t), intent(inout) :: target
-    character(:), allocatable, intent(in) :: linking
 
-    integer :: i, j, ilib
+!> Topologically sort a target for scheduling by 
+!>  recursing over its dependencies.
+!> 
+!> Checks disk-cached source hashes to determine if objects are
+!>  up-to-date. Up-to-date sources are tagged as skipped.
+!>
+recursive subroutine sort_target(target)
+    type(build_target_t), intent(inout), target :: target
+
+    integer :: i, j, fh, stat
     type(build_target_t), pointer :: exe_obj
-    character(:), allocatable :: objs, link_flags
 
-    if (target%built) then
+    ! Check if target has already been processed (as a dependency)
+    if (target%sorted .or. target%skip) then
         return
     end if
 
+    ! Check for a circular dependency
+    ! (If target has been touched but not processed)
     if (target%touched) then
         write(*,*) '(!) Circular dependency found with: ',target%output_file
         stop
     else
-        target%touched = .true.
+        target%touched = .true.  ! Set touched flag
     end if
 
-    objs = " "
+    ! Load cached source file digest if present
+    if (.not.allocated(target%digest_cached) .and. &
+         exists(target%output_file) .and. &
+         exists(target%output_file//'.digest')) then
 
-    do i=1,size(target%dependencies)
+        allocate(target%digest_cached)
+        open(newunit=fh,file=target%output_file//'.digest',status='old')
+        read(fh,*,iostat=stat) target%digest_cached
+        close(fh)
 
-        if (associated(target%dependencies(i)%ptr)) then
-            call build_target(model,target%dependencies(i)%ptr,linking)
+        if (stat /= 0) then    ! Cached digest is not recognized
+            deallocate(target%digest_cached)
         end if
 
-        if (target%target_type == FPM_TARGET_ARCHIVE ) then
+    end if
+
+    if (allocated(target%source)) then
 
-            ! Construct object list for archive
-            objs = objs//" "//target%dependencies(i)%ptr%output_file
+        ! Skip if target is source-based and source file is unmodified
+        if (allocated(target%digest_cached)) then
+            if (target%digest_cached == target%source%digest) target%skip = .true.
+        end if
 
-        else if (target%target_type == FPM_TARGET_EXECUTABLE .and. &
-                 target%dependencies(i)%ptr%target_type ==  FPM_TARGET_OBJECT) then
+    elseif (exists(target%output_file)) then
 
-            exe_obj => target%dependencies(i)%ptr
-                
-            ! Construct object list for executable
-            objs = " "//exe_obj%output_file
-                
-            ! Include non-library object dependencies
-            do j=1,size(exe_obj%dependencies)
+        ! Skip if target is not source-based and already exists
+        target%skip = .true.
 
-                if (allocated(exe_obj%dependencies(j)%ptr%source)) then
-                    if (exe_obj%dependencies(j)%ptr%source%unit_scope == exe_obj%source%unit_scope) then
-                        objs = objs//" "//exe_obj%dependencies(j)%ptr%output_file
-                    end if
-                end if
+    end if
 
-            end do
+    ! Loop over target dependencies
+    target%schedule = 1
+    do i=1,size(target%dependencies)
+
+        ! Sort dependency
+        call sort_target(target%dependencies(i)%ptr)
+
+        if (.not.target%dependencies(i)%ptr%skip) then
+
+            ! Can't skip target if any dependency is not skipped
+            target%skip = .false.
+
+            ! Set target schedule after all of its dependencies
+            target%schedule = max(target%schedule,target%dependencies(i)%ptr%schedule+1)
 
         end if
 
     end do
 
+    ! Mark flag as processed: either sorted or skipped
+    target%sorted = .not.target%skip
+
+end subroutine sort_target
+
+
+!> Construct a build schedule from the sorted targets.
+!>
+!> The schedule is broken into regions, described by `schedule_ptr`,
+!>  where targets in each region can be compiled in parallel.
+!>
+subroutine schedule_targets(queue, schedule_ptr, targets)
+    type(build_target_ptr), allocatable, intent(out) :: queue(:)
+    integer, allocatable :: schedule_ptr(:)
+    type(build_target_ptr), intent(in) :: targets(:)
+
+    integer :: i, j
+    integer :: n_schedule, n_sorted
+
+    n_schedule = 0   ! Number of schedule regions
+    n_sorted = 0     ! Total number of targets to build
+    do i=1,size(targets)
+
+        if (targets(i)%ptr%sorted) then
+            n_sorted = n_sorted + 1
+        end if
+        n_schedule = max(n_schedule, targets(i)%ptr%schedule)
+
+    end do
+
+    allocate(queue(n_sorted))
+    allocate(schedule_ptr(n_schedule+1))
+
+    ! Construct the target queue and schedule region pointer
+    n_sorted = 1
+    schedule_ptr(n_sorted) = 1
+    do i=1,n_schedule
+
+        do j=1,size(targets)
+
+            if (targets(j)%ptr%sorted) then
+                if (targets(j)%ptr%schedule == i) then
+
+                    queue(n_sorted)%ptr => targets(j)%ptr
+                    n_sorted = n_sorted + 1
+                end if
+            end if
+
+        end do
+
+        schedule_ptr(i+1) = n_sorted
+
+    end do
+
+end subroutine schedule_targets
+
+
+!> Call compile/link command for a single target.
+!>
+!> If successful, also caches the source file digest to disk.
+!>
+subroutine build_target(model,target)
+    type(fpm_model_t), intent(in) :: model
+    type(build_target_t), intent(in), target :: target
+
+    integer :: ilib, fh
+    character(:), allocatable :: link_flags
+
     if (.not.exists(dirname(target%output_file))) then
         call mkdir(dirname(target%output_file))
     end if
@@ -119,22 +208,34 @@ recursive subroutine build_target(model,target,linking)
               // " -o " // target%output_file)
 
     case (FPM_TARGET_EXECUTABLE)
-        link_flags = linking
+
+        link_flags = string_cat(target%link_objects," ") 
+
+        if (allocated(model%library_file)) then
+            link_flags = link_flags//" "//model%library_file//" "//model%link_flags
+        else
+            link_flags = link_flags//" "//model%link_flags
+        end if
+        
         if (allocated(target%link_libraries)) then
-            do ilib = 1, size(target%link_libraries)
-                link_flags = link_flags // " -l" // target%link_libraries(ilib)%s
-            end do
+            if (size(target%link_libraries) > 0) then
+                link_flags = link_flags // " -l" // string_cat(target%link_libraries," -l")
+            end if
         end if
-
-        call run("gfortran " // objs // model%fortran_compile_flags &
-              //link_flags// " -o " // target%output_file)
+        
+        call run("gfortran " // model%fortran_compile_flags &
+              //" "//link_flags// " -o " // target%output_file)
 
     case (FPM_TARGET_ARCHIVE)
-        call run("ar -rs " // target%output_file // objs)
+        call run("ar -rs " // target%output_file // " " // string_cat(target%link_objects," "))
 
     end select
 
-    target%built = .true.
+    if (allocated(target%source)) then
+        open(newunit=fh,file=target%output_file//'.digest',status='unknown')
+        write(fh,*) target%source%digest
+        close(fh)
+    end if
 
 end subroutine build_target
 

fpm/src/fpm_model.f90

@@ -1,5 +1,6 @@
 module fpm_model
 ! Definition and validation of the backend model
+use iso_fortran_env, only: int64
 use fpm_strings, only: string_t
 implicit none
 
@@ -53,6 +54,8 @@ module fpm_model
         ! Files INCLUDEd by this source file
     type(string_t), allocatable :: link_libraries(:)
         ! Native libraries to link against
+    integer(int64) :: digest
+        ! Current hash
 end type srcfile_t
 
 type build_target_ptr
@@ -70,9 +73,20 @@ module fpm_model
     integer :: target_type = FPM_TARGET_UNKNOWN
     type(string_t), allocatable :: link_libraries(:)
         ! Native libraries to link against
+    type(string_t), allocatable :: link_objects(:)
+        ! Objects needed to link this target
 
-    logical :: built = .false.
     logical :: touched = .false.
+        ! Flag set when first visited to check for circular dependencies
+    logical :: sorted = .false.
+        ! Flag set if build target is sorted for building
+    logical :: skip = .false.
+        ! Flag set if build target will be skipped (not built)
+
+    integer :: schedule = -1
+        ! Targets in the same schedule group are guaranteed to be independent
+    integer(int64), allocatable :: digest_cached
+        ! Previous hash
 
 end type build_target_t
 
@@ -89,6 +103,8 @@ module fpm_model
         ! Command line flags passed to fortran for compilation
     character(:), allocatable :: link_flags
         ! Command line flags pass for linking
+    character(:), allocatable :: library_file
+        ! Output file for library archive
     character(:), allocatable :: output_directory
         ! Base directory for build
     type(string_t), allocatable :: link_libraries(:)