Skip to content

Commit f1a6807

Browse files
johnscancellajohnscancella
committed
changed .bagit to be version 2.0 and implemented 1.0 spec that all manifest must list all files
1 parent 1e67c03 commit f1a6807

28 files changed

Lines changed: 132 additions & 42 deletions

‎src/integration/java/gov/loc/repository/bagit/ReaderWriterVerifierIntegrationTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,9 +91,9 @@ public void testReaderWriterVersion97() throws Exception{
9191
}
9292

9393
@Test
94-
public void testReaderWriterVersion98() throws Exception{
94+
public void testReaderWriterVersion2_0() throws Exception{
9595
BagReader reader = new BagReader();
96-
Path rootDir = Paths.get(this.getClass().getClassLoader().getResource("bags/v0_98/bag").toURI());
96+
Path rootDir = Paths.get(this.getClass().getClassLoader().getResource("bags/v2_0/bag").toURI());
9797
Bag bag = reader.read(rootDir);
9898
Path outputDir = Paths.get(folder.newFolder().toURI());
9999

‎src/main/java/gov/loc/repository/bagit/creator/BagCreator.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ public static Bag bagInPlace(final Path root, final Collection<SupportedAlgorith
8181
*/
8282
@Incubating
8383
public static Bag createDotBagit(final Path root, final Collection<SupportedAlgorithm> algorithms, final boolean includeHidden) throws NoSuchAlgorithmException, IOException{
84-
final Bag bag = new Bag(new Version(0, 98));
84+
final Bag bag = new Bag(new Version(2, 0));
8585
bag.setRootDir(root);
8686
logger.info("Creating a bag with version: [{}] in directory: [{}]", bag.getVersion(), root);
8787

‎src/main/java/gov/loc/repository/bagit/verify/AbstractPayloadFileExistsInManifestsVistor.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package gov.loc.repository.bagit.verify;
2+
3+
import java.io.IOException;
4+
import java.nio.file.FileVisitResult;
5+
import java.nio.file.Files;
6+
import java.nio.file.Path;
7+
import java.nio.file.Paths;
8+
import java.nio.file.SimpleFileVisitor;
9+
import java.nio.file.attribute.BasicFileAttributes;
10+
11+
import org.slf4j.Logger;
12+
import org.slf4j.LoggerFactory;
13+
14+
/**
15+
* Implements {@link SimpleFileVisitor} to ensure that the encountered file is in one of the manifests.
16+
*/
17+
abstract public class AbstractPayloadFileExistsInManifestsVistor extends SimpleFileVisitor<Path> {
18+
protected static final Logger logger = LoggerFactory.getLogger(AbstractPayloadFileExistsInManifestsVistor.class);
19+
protected transient final boolean ignoreHiddenFiles;
20+
21+
public AbstractPayloadFileExistsInManifestsVistor(final boolean ignoreHiddenFiles) {
22+
this.ignoreHiddenFiles = ignoreHiddenFiles;
23+
}
24+
25+
@Override
26+
public FileVisitResult preVisitDirectory(final Path dir, final BasicFileAttributes attrs) throws IOException {
27+
if(ignoreHiddenFiles && Files.isHidden(dir) || dir.endsWith(Paths.get(".bagit"))){
28+
logger.debug("Skipping [{}] cause it is a hidden folder", dir);
29+
return FileVisitResult.SKIP_SUBTREE;
30+
}
31+
32+
return FileVisitResult.CONTINUE;
33+
}
34+
}

‎src/main/java/gov/loc/repository/bagit/verify/BagVerifier.java‎

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
/**
4444
* Responsible for verifying if a bag is valid, complete
4545
*/
46+
@SuppressWarnings({"PMD.GodClass"}) //TODO refactor
4647
public final class BagVerifier {
4748
private static final Logger logger = LoggerFactory.getLogger(BagVerifier.class);
4849

@@ -237,14 +238,20 @@ public void isComplete(final Bag bag, final boolean ignoreHiddenFiles) throws
237238

238239
final Set<Path> allFilesListedInManifests = getAllFilesListedInManifests(bag);
239240
checkAllFilesListedInManifestExist(allFilesListedInManifests);
240-
checkAllFilesInPayloadDirAreListedInAManifest(allFilesListedInManifests, dataDir, ignoreHiddenFiles);
241+
242+
if(new Version(1,0).compareTo(bag.getVersion()) < 0){
243+
checkAllFilesInPayloadDirAreListedInAtLeastOneAManifest(allFilesListedInManifests, dataDir, ignoreHiddenFiles);
244+
}
245+
else{
246+
CheckAllFilesInPayloadDirAreListedInAllManifests(bag.getPayLoadManifests(), dataDir, ignoreHiddenFiles);
247+
}
241248
}
242249

243250
/*
244251
* Get the directory that contains the payload files.
245252
*/
246253
private Path getDataDir(final Bag bag){
247-
if(bag.getVersion().compareTo(new Version(0, 98)) >= 0){ //is it a .bagit version?
254+
if(bag.getVersion().compareTo(new Version(2, 0)) >= 0){ //is it a .bagit version?
248255
return bag.getRootDir();
249256
}
250257

@@ -271,7 +278,7 @@ private void checkBagitFileExists(final Path rootDir, final Version version) thr
271278
logger.info("Checking if bagit.txt file exists");
272279
Path bagitFile = rootDir.resolve("bagit.txt");
273280
//@Incubating
274-
if(version.compareTo(new Version(0, 98)) >= 0){ //is it a .bagit version?
281+
if(version.compareTo(new Version(2, 0)) >= 0){ //is it a .bagit version?
275282
bagitFile = rootDir.resolve(DOT_BAGIT_DIR_NAME + File.separator + "bagit.txt");
276283
}
277284

@@ -301,7 +308,7 @@ private void checkIfAtLeastOnePayloadManifestsExist(final Path rootDir, final Ve
301308

302309
DirectoryStream<Path> directoryStream = Files.newDirectoryStream(rootDir);
303310
//@Incubating
304-
if(version.compareTo(new Version(0, 98)) >= 0){ //is it a .bagit version?
311+
if(version.compareTo(new Version(2, 00)) >= 0){ //is it a .bagit version?
305312
directoryStream = Files.newDirectoryStream(rootDir.resolve(DOT_BAGIT_DIR_NAME));
306313
}
307314

@@ -327,7 +334,7 @@ private Set<Path> getAllFilesListedInManifests(final Bag bag) throws IOException
327334

328335
DirectoryStream<Path> directoryStream = Files.newDirectoryStream(bag.getRootDir());
329336
//@Incubating
330-
if(bag.getVersion().compareTo(new Version(0, 98)) >= 0){ //is it a .bagit version?
337+
if(bag.getVersion().compareTo(new Version(2, 00)) >= 0){ //is it a .bagit version?
331338
directoryStream = Files.newDirectoryStream(bag.getRootDir().resolve(DOT_BAGIT_DIR_NAME));
332339
}
333340

@@ -370,10 +377,20 @@ private void checkAllFilesListedInManifestExist(final Set<Path> files) throws Fi
370377
/*
371378
* Make sure all files in the directory are in at least 1 manifest
372379
*/
373-
private void checkAllFilesInPayloadDirAreListedInAManifest(final Set<Path> filesListedInManifests, final Path payloadDir, final boolean ignoreHiddenFiles) throws IOException{
380+
private void checkAllFilesInPayloadDirAreListedInAtLeastOneAManifest(final Set<Path> filesListedInManifests, final Path payloadDir, final boolean ignoreHiddenFiles) throws IOException{
374381
logger.debug("Checking if all payload files (files in {} dir) are listed in at least one manifest", payloadDir);
375382
if(Files.exists(payloadDir)){
376-
Files.walkFileTree(payloadDir, new PayloadFileExistsInManifestVistor(filesListedInManifests, ignoreHiddenFiles));
383+
Files.walkFileTree(payloadDir, new PayloadFileExistsInAtLeastOneManifestVistor(filesListedInManifests, ignoreHiddenFiles));
384+
}
385+
}
386+
387+
/*
388+
* as per the bagit-spec 1.0+ all files have to be listed in all manifests
389+
*/
390+
private void CheckAllFilesInPayloadDirAreListedInAllManifests(final Set<Manifest> payLoadManifests, final Path payloadDir, final boolean ignoreHiddenFiles) throws IOException{
391+
logger.debug("Checking if all payload files (files in {} dir) are listed in all manifests", payloadDir);
392+
if(Files.exists(payloadDir)){
393+
Files.walkFileTree(payloadDir, new PayloadFileExistsInAllManifestsVistor(payLoadManifests, ignoreHiddenFiles));
377394
}
378395
}
379396

‎src/main/java/gov/loc/repository/bagit/verify/PayloadFileExistsInAllManifestsVistor.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
package gov.loc.repository.bagit.verify;
2+
3+
import java.nio.file.FileVisitResult;
4+
import java.nio.file.Files;
5+
import java.nio.file.Path;
6+
import java.nio.file.SimpleFileVisitor;
7+
import java.nio.file.attribute.BasicFileAttributes;
8+
import java.util.Set;
9+
10+
import gov.loc.repository.bagit.domain.Manifest;
11+
import gov.loc.repository.bagit.exceptions.FileNotInManifestException;
12+
13+
/**
14+
* Implements {@link SimpleFileVisitor} to ensure that the encountered file is in one of the manifests.
15+
*/
16+
public class PayloadFileExistsInAllManifestsVistor extends AbstractPayloadFileExistsInManifestsVistor {
17+
private transient final Set<Manifest> manifests;
18+
19+
public PayloadFileExistsInAllManifestsVistor(final Set<Manifest> manifests, final boolean ignoreHiddenFiles) {
20+
super(ignoreHiddenFiles);
21+
this.manifests = manifests;
22+
}
23+
24+
@Override
25+
public FileVisitResult visitFile(final Path path, final BasicFileAttributes attrs)throws FileNotInManifestException{
26+
if(Files.isRegularFile(path)){
27+
for(final Manifest manifest : manifests){
28+
if(!manifest.getFileToChecksumMap().keySet().contains(path.normalize())){
29+
throw new FileNotInManifestException("File " + path + " is in the payload directory but isn't listed in manifest manifest-" + manifest.getAlgorithm().getBagitName() + ".txt");
30+
}
31+
}
32+
}
33+
logger.debug("[{}] is in all manifests", path);
34+
return FileVisitResult.CONTINUE;
35+
}
36+
}

src/main/java/gov/loc/repository/bagit/verify/PayloadFileExistsInManifestVistor.java renamed to src/main/java/gov/loc/repository/bagit/verify/PayloadFileExistsInAtLeastOneManifestVistor.java

Lines changed: 3 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,23 @@
11
package gov.loc.repository.bagit.verify;
22

3-
import java.io.IOException;
43
import java.nio.file.FileVisitResult;
54
import java.nio.file.Files;
65
import java.nio.file.Path;
7-
import java.nio.file.Paths;
86
import java.nio.file.SimpleFileVisitor;
97
import java.nio.file.attribute.BasicFileAttributes;
108
import java.util.Set;
119

12-
import org.slf4j.Logger;
13-
import org.slf4j.LoggerFactory;
14-
1510
import gov.loc.repository.bagit.exceptions.FileNotInManifestException;
1611

1712
/**
1813
* Implements {@link SimpleFileVisitor} to ensure that the encountered file is in one of the manifests.
1914
*/
20-
public class PayloadFileExistsInManifestVistor extends SimpleFileVisitor<Path> {
21-
private static final Logger logger = LoggerFactory.getLogger(PayloadFileExistsInManifestVistor.class);
15+
public class PayloadFileExistsInAtLeastOneManifestVistor extends AbstractPayloadFileExistsInManifestsVistor {
2216
private transient final Set<Path> filesListedInManifests;
23-
private transient final boolean ignoreHiddenFiles;
2417

25-
public PayloadFileExistsInManifestVistor(final Set<Path> filesListedInManifests, final boolean ignoreHiddenFiles) {
18+
public PayloadFileExistsInAtLeastOneManifestVistor(final Set<Path> filesListedInManifests, final boolean ignoreHiddenFiles) {
19+
super(ignoreHiddenFiles);
2620
this.filesListedInManifests = filesListedInManifests;
27-
this.ignoreHiddenFiles = ignoreHiddenFiles;
28-
}
29-
30-
@Override
31-
public FileVisitResult preVisitDirectory(final Path dir, final BasicFileAttributes attrs) throws IOException {
32-
if(ignoreHiddenFiles && Files.isHidden(dir) || dir.endsWith(Paths.get(".bagit"))){
33-
logger.debug("Skipping [{}] cause it is a hidden folder", dir);
34-
return FileVisitResult.SKIP_SUBTREE;
35-
}
36-
37-
return FileVisitResult.CONTINUE;
3821
}
3922

4023
@Override

‎src/main/java/gov/loc/repository/bagit/writer/BagWriter.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
@SuppressWarnings("PMD.TooManyMethods")
3232
public final class BagWriter {
3333
private static final Logger logger = LoggerFactory.getLogger(BagWriter.class);
34-
private static final Version VERSION_0_98 = new Version(0, 98);
34+
private static final Version VERSION_2_0 = new Version(2, 0);
3535
private static final Version VERSION_0_95 = new Version(0, 95);
3636

3737
private BagWriter(){
@@ -76,7 +76,7 @@ public static void write(final Bag bag, final Path outputDir) throws IOException
7676
private static Path writeVersionDependentPayloadFiles(final Bag bag, final Path outputDir) throws IOException{
7777
Path bagitDir = outputDir;
7878
//@Incubating
79-
if(VERSION_0_98.compareTo(bag.getVersion()) <= 0){
79+
if(VERSION_2_0.compareTo(bag.getVersion()) <= 0){
8080
bagitDir = outputDir.resolve(".bagit");
8181
Files.createDirectories(bagitDir);
8282
writePayloadFiles(bag.getPayLoadManifests(), outputDir, bag.getRootDir());

‎src/test/java/gov/loc/repository/bagit/creator/BagCreatorTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ public void testCreateDotBagit() throws IOException, NoSuchAlgorithmException{
8181
Path expectedBagitFile = dotbagitDir.resolve("bagit.txt");
8282

8383
Bag bag = BagCreator.createDotBagit(rootFolderPath, Arrays.asList(StandardSupportedAlgorithms.MD5), false);
84-
assertEquals(new Version(0, 98), bag.getVersion());
84+
assertEquals(new Version(2, 0), bag.getVersion());
8585

8686
assertTrue(Files.exists(expectedBagitFile));
8787
assertTrue(Files.exists(expectedManifestFile));

‎src/test/java/gov/loc/repository/bagit/reader/BagReaderTest.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -265,15 +265,15 @@ public void testReadVersion0_97Bag() throws Exception{
265265
}
266266

267267
@Test
268-
public void testReadVersion0_98Bag() throws Exception{
269-
Path rootBag = Paths.get(getClass().getClassLoader().getResource("bags/v0_98/bag").toURI());
268+
public void testReadVersion2_0Bag() throws Exception{
269+
Path rootBag = Paths.get(getClass().getClassLoader().getResource("bags/v2_0/bag").toURI());
270270
Path[] payloadFiles = new Path[]{rootBag.resolve("dir1/test3.txt"), rootBag.resolve("dir2/dir3/test5.txt"),
271271
rootBag.resolve("dir2/test4.txt"), rootBag.resolve("test1.txt"), rootBag.resolve("test2.txt")};
272272

273273
Bag returnedBag = sut.read(rootBag);
274274

275275
assertNotNull(returnedBag);
276-
assertEquals(new Version(0, 98), returnedBag.getVersion());
276+
assertEquals(new Version(2, 0), returnedBag.getVersion());
277277
Manifest payloadManifest = (Manifest) returnedBag.getPayLoadManifests().toArray()[0];
278278
for(Path payloadFile : payloadFiles){
279279
assertTrue("payload manifest should contain " + payloadFile, payloadManifest.getFileToChecksumMap().containsKey(payloadFile));

‎src/test/java/gov/loc/repository/bagit/verify/BagVerifierTest.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -118,8 +118,8 @@ public void testVersion0_97IsValid() throws Exception{
118118
}
119119

120120
@Test
121-
public void testVersion0_98IsValid() throws Exception{
122-
rootDir = Paths.get(new File("src/test/resources/bags/v0_98/bag").toURI());
121+
public void testVersion2_0IsValid() throws Exception{
122+
rootDir = Paths.get(new File("src/test/resources/bags/v2_0/bag").toURI());
123123
Bag bag = reader.read(rootDir);
124124

125125
sut.isValid(bag, true);
@@ -240,6 +240,13 @@ public void testAddSHA3SupportViaExtension() throws Exception{
240240
extendedSut.isValid(bag, true);
241241
}
242242

243+
@Test(expected=FileNotInManifestException.class)
244+
public void testNotALlFilesListedInAllManifestsThrowsException() throws Exception{
245+
Path bagDir = Paths.get(new File("src/test/resources/notAllFilesListedInAllManifestsBag").toURI());
246+
Bag bag = reader.read(bagDir);
247+
sut.isValid(bag, true);
248+
}
249+
243250
/*
244251
* Technically valid but highly discouraged
245252
*/

0 commit comments

Comments
 (0)