diff --git a/.github/workflows/build-ubuntu.yaml b/.github/workflows/build-ubuntu.yaml index 98b8c76824..4e655ec58a 100644 --- a/.github/workflows/build-ubuntu.yaml +++ b/.github/workflows/build-ubuntu.yaml @@ -224,7 +224,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Send e-mail on failed run - if: failure() && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) + # Branch-local: failures are expected on this branch; do not notify. + if: false # freeze to v17 uses: dawidd6/action-send-mail@42942bc2f8fba4e611b459a018967a6a7c78c68c with: @@ -247,8 +248,18 @@ jobs: needs: build-package strategy: matrix: - flavor: ["normal", "crippled-tmp", "crippled-home", "nfs-home", "custom-config1"] + # Branch-local (diagnostic branch, never merged into master): + # only the flavor that hits the intermittent LinkAnnexFailed on NFS, + # run several times per invocation since a single run reproduces it + # in roughly two nights out of three. + flavor: ["nfs-home"] os: [ubuntu-24.04] + # Attribute-cache experiment: is the LinkAnnexFailed mtime jitter + # caused by NFS client attribute caching? default = what the + # nightlies run, actimeo0 = attribute caching off only, noac = + # that plus synchronous writes (the usual recommendation). + acmode: [default, actimeo0, noac] + rep: [1, 2, 3] fail-fast: false steps: - name: Checkout this repository @@ -298,7 +309,14 @@ jobs: echo "/tmp/nfsmount_ localhost(rw)" | sudo bash -c 'cat - > /etc/exports' sudo apt-get install -y nfs-kernel-server sudo exportfs -a - sudo mount -t nfs localhost:/tmp/nfsmount_ /tmp/nfsmount + case "${{ matrix.acmode }}" in + noac) mountopts=(-o noac) ;; + actimeo0) mountopts=(-o actimeo=0) ;; + *) mountopts=() ;; + esac + sudo mount -t nfs "${mountopts[@]}" localhost:/tmp/nfsmount_ /tmp/nfsmount + echo "NFS mount options in effect (acmode=${{ matrix.acmode }}):" + findmnt -no OPTIONS /tmp/nfsmount fi case "${{ matrix.flavor }}" in @@ -345,7 +363,10 @@ jobs: cd $HOME export | grep -e crippledfs || : - timeout 3600 git annex test "${test_opts[@]:-}" + # Branch-local: noac writes synchronously and is much slower, + # so give the suite room rather than have it die at the timeout + # and leave the experiment unreadable. + timeout 7200 git annex test "${test_opts[@]:-}" - name: Set final PR status if: always() && github.event.inputs.pr != '' @@ -359,7 +380,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Send e-mail on failed run - if: failure() && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) + # Branch-local: failures are expected on this branch; do not notify. + if: false # freeze to v17 uses: dawidd6/action-send-mail@42942bc2f8fba4e611b459a018967a6a7c78c68c with: @@ -378,6 +400,8 @@ jobs: See for more information. test-annex-more: + # Branch-local: not needed for the NFS diagnostic runs. + if: false runs-on: ubuntu-24.04 needs: build-package steps: @@ -428,6 +452,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} test-datalad: + # Branch-local: not needed for the NFS diagnostic runs. + if: false runs-on: ubuntu-24.04 needs: build-package strategy: diff --git a/patches/20260922-f68b252dbd-diag-linkannex-failure.patch b/patches/20260922-f68b252dbd-diag-linkannex-failure.patch new file mode 100644 index 0000000000..186c2da391 --- /dev/null +++ b/patches/20260922-f68b252dbd-diag-linkannex-failure.patch @@ -0,0 +1,126 @@ +Description: CI diagnostic: report why linkAnnex failed + The nightly `test-annex (nfs-home)` job intermittently fails with + " failed to link to annex" (git-annex add of an unlocked file) or + "unlock failed", in a different test each night. Both messages come + from linkAnnex returning LinkAnnexFailed, which is reachable by two + paths that say nothing at all about what went wrong: + . + * linkOrCopy returned Nothing - either an IOException was swallowed by + catchDefaultIO/catchBoolIO, or cp / the chmod in preserveGitMode + returned False; + * the source file's inode cache (inode, size, high resolution mtime) + did not compare equal before and after the copy, so checksrcunchanged + deleted the destination. + . + The CI transcripts cannot distinguish these, which makes the NFS + failures impossible to diagnose after the fact. This patch adds a + warning to each of those paths, including the before/after inode caches, + so that the next failing run says which one it is. + . + Diagnostic only - no behaviour change beyond the extra messages on + paths that already fail. Drop it once the cause is understood. + . + Drafted with Claude Code assistance. +Origin: vendor, https://github.com/con/git-annex +Author: Yaroslav Halchenko +Forwarded: not-needed +Last-Update: 2026-09-22 +SPDX-FileCopyrightText: 2026 Yaroslav Halchenko +SPDX-License-Identifier: AGPL-3.0-or-later +--- +diff --git a/Annex/Content/LowLevel.hs b/Annex/Content/LowLevel.hs +index 49fc442..0a4daa9 100644 +--- a/Annex/Content/LowLevel.hs ++++ b/Annex/Content/LowLevel.hs +@@ -47,12 +47,16 @@ linkOrCopy :: Key -> OsPath -> OsPath -> Maybe FileMode -> Annex (Maybe LinkedOr + linkOrCopy = linkOrCopy' (annexThin <$> Annex.getGitConfig) + + linkOrCopy' :: Annex Bool -> Key -> OsPath -> OsPath -> Maybe FileMode -> Annex (Maybe LinkedOrCopied) +-linkOrCopy' canhardlink key src dest destmode = catchDefaultIO Nothing $ +- ifM canhardlink ++linkOrCopy' canhardlink key src dest destmode = go `catchIO` reportfailure ++ where ++ go = ifM canhardlink + ( hardlinkorcopy + , copy =<< getstat + ) +- where ++ reportfailure e = do ++ warning $ UnquotedString $ ++ "DIAGNOSTIC: linkOrCopy threw: " ++ show e ++ return Nothing + hardlinkorcopy = do + s <- getstat + if linkCount s > 1 +@@ -75,14 +79,29 @@ checkedCopyFile key src dest destmode = catchBoolIO $ + =<< liftIO (R.getFileStatus (fromOsPath src)) + + checkedCopyFile' :: Key -> OsPath -> OsPath -> Maybe FileMode -> FileStatus -> Annex Bool +-checkedCopyFile' key src dest destmode s = catchBoolIO $ do +- sz <- liftIO $ getFileSize' src s +- ifM (checkDiskSpace' sz (Just $ takeDirectory dest) key 0 True) +- ( liftIO $ +- copyFileExternal CopyAllMetaData src dest +- <&&> preserveGitMode dest destmode +- , return False +- ) ++checkedCopyFile' key src dest destmode s = go `catchIO` reportfailure ++ where ++ go = do ++ sz <- liftIO $ getFileSize' src s ++ ifM (checkDiskSpace' sz (Just $ takeDirectory dest) key 0 True) ++ ( ifM (liftIO $ copyFileExternal CopyAllMetaData src dest) ++ ( ifM (liftIO $ preserveGitMode dest destmode) ++ ( return True ++ , do ++ diagnostic "preserveGitMode failed" ++ return False ++ ) ++ , do ++ diagnostic "copyFileExternal failed" ++ return False ++ ) ++ , return False ++ ) ++ reportfailure e = do ++ diagnostic ("exception: " ++ show e) ++ return False ++ diagnostic m = warning $ UnquotedString $ ++ "DIAGNOSTIC: checkedCopyFile: " ++ m + + preserveGitMode :: OsPath -> Maybe FileMode -> IO Bool + preserveGitMode f (Just mode) +diff --git a/Annex/Content.hs b/Annex/Content.hs +index e3b2202..530b70b 100644 +--- a/Annex/Content.hs ++++ b/Annex/Content.hs +@@ -631,7 +631,12 @@ linkAnnex fromto key src (Just srcic) dest destmode = + else Database.Keys.addInodeCaches key [srcic] + return LinkAnnexNoop + Nothing -> linkOrCopy key src dest destmode >>= \case +- Nothing -> failed ++ Nothing -> do ++ warning $ "DIAGNOSTIC: linkAnnex: linkOrCopy failed for " ++ <> QuotedPath src ++ <> " => " ++ <> QuotedPath dest ++ failed + Just r -> do + case fromto of + From -> thawContent dest +@@ -649,7 +654,14 @@ linkAnnex fromto key src (Just srcic) dest destmode = + Database.Keys.addInodeCaches key $ + catMaybes [destic, Just srcic] + return LinkAnnexOk +- _ -> do ++ msrcic' -> do ++ warning $ "DIAGNOSTIC: linkAnnex: inode cache of " ++ <> QuotedPath src ++ <> UnquotedString (" changed while copying it; before: " ++ ++ showInodeCache srcic ++ ++ "; after: " ++ ++ maybe "(missing or not a regular file)" ++ showInodeCache msrcic') + liftIO $ removeWhenExistsWith removeFile dest + failed +