Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: Bake faster #728

Open
wants to merge 9 commits into
base: master
Choose a base branch
from
Open

WIP: Bake faster #728

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
7f0f8e2
run cuncurrent rendering
ancho May 1, 2020
a85f679
run concurrent crawl and asset copying
ancho May 2, 2020
4658088
synchronize query access add index for tags
ancho May 2, 2020
57d3707
fix renderCustomTypePaper test for GroovyMarkupTemplateEngineRenderin…
ancho May 10, 2020
825d497
fix tests for windows with hidden files
ancho May 10, 2020
92ad9eb
show how long each task took
ancho May 23, 2021
5df8e48
cleanup crawler to execute data folder processing concurrently
ancho May 23, 2021
d26cc96
jdk11 source and traget compatibility
ancho May 23, 2021
973a294
fix reading from relative template paths
ancho Sep 27, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,8 @@ subprojects {

// add source and target compatibility for all JavaCompile tasks
tasks.withType(JavaCompile) {
sourceCompatibility = 1.8
targetCompatibility = 1.8
sourceCompatibility = 11
targetCompatibility = 11
}

test {
Expand Down
30 changes: 16 additions & 14 deletions jbake-core/src/main/java/org/jbake/app/Asset.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
Expand Down Expand Up @@ -53,7 +58,9 @@ public Asset(JBakeConfiguration config) {
* read from configuration
*/
public void copy() {
copy(config.getAssetFolder());
if (config.getAssetFolder() != null) {
copy(config.getAssetFolder());
}
}

/**
Expand All @@ -62,12 +69,7 @@ public void copy() {
* @param path The starting path
*/
public void copy(File path) {
FileFilter filter = new FileFilter() {
@Override
public boolean accept(File file) {
return (!config.getAssetIgnoreHidden() || !file.isHidden()) && (file.isFile() || FileUtil.directoryOnlyIfNotIgnored(file, config));
}
};
FileFilter filter = file -> (!config.getAssetIgnoreHidden() || !file.isHidden()) && (file.isFile() || FileUtil.directoryOnlyIfNotIgnored(file, config));
copy(path, config.getDestinationFolder(), filter);
}

Expand All @@ -78,7 +80,7 @@ public boolean accept(File file) {
*/
public void copySingleFile(File asset) {
try {
if ( !asset.isDirectory() ) {
if (!asset.isDirectory()) {
String targetPath = config.getDestinationFolder().getCanonicalPath() + File.separatorChar + assetSubPath(asset);
LOGGER.info("Copying single asset file to [{}]", targetPath);
copyFile(asset, new File(targetPath));
Expand All @@ -92,14 +94,15 @@ public void copySingleFile(File asset) {

/**
* Determine if a given file is an asset file.
*
* @param path to the file to validate.
* @return true if the path provided points to a file in the asset folder.
*/
public boolean isAssetFile(File path) {
boolean isAsset = false;

try {
if(FileUtil.directoryOnlyIfNotIgnored(path.getParentFile(), config)) {
if (FileUtil.directoryOnlyIfNotIgnored(path.getParentFile(), config)) {
if (FileUtil.isFileInDirectory(path, config.getAssetFolder())) {
isAsset = true;
} else if (FileUtil.isFileInDirectory(path, config.getContentFolder())
Expand Down Expand Up @@ -140,17 +143,16 @@ private String assetSubPath(File asset) throws IOException {
}

private void copy(File sourceFolder, File targetFolder, final FileFilter filter) {
final File[] assets = sourceFolder.listFiles(filter);
if (assets != null) {
Arrays.sort(assets);
for (File asset : assets) {
File[] array = sourceFolder.listFiles(filter);
if ( array != null ) {
Arrays.stream(array).parallel().forEach(asset -> {
final File target = new File(targetFolder, asset.getName());
if (asset.isFile()) {
copyFile(asset, target);
} else if (asset.isDirectory()) {
copy(asset, target, filter);
}
}
});
}
}

Expand Down
22 changes: 11 additions & 11 deletions jbake-core/src/main/java/org/jbake/app/ContentStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,11 +148,6 @@ public void close() {
DBUtil.closeDataStore();
}

public void shutdown() {

// Orient.instance().shutdown();
}

private void startupIfEnginesAreMissing() {
// Using a jdk which doesn't bundle a javascript engine
// throws a NoClassDefFoundError while logging the warning
Expand All @@ -169,8 +164,6 @@ private void startupIfEnginesAreMissing() {

public void drop() {
activateOnCurrentThread();
// db.drop();

orient.drop(name);
}

Expand Down Expand Up @@ -290,21 +283,23 @@ private void insertTemplatesSignature(String currentTemplatesSignature) {
executeCommand(STATEMENT_INSERT_TEMPLATES_SIGNATURE, currentTemplatesSignature);
}

private DocumentList<DocumentModel> query(String sql) {
private synchronized DocumentList<DocumentModel> query(String sql) {
activateOnCurrentThread();
OResultSet results = db.query(sql);
return DocumentList.wrap(results);
}

private DocumentList<DocumentModel> query(String sql, Object... args) {
private synchronized DocumentList<DocumentModel> query(String sql, Object... args) {
activateOnCurrentThread();
OResultSet results = db.command(sql, args);
return DocumentList.wrap(results);
}

private void executeCommand(String query, Object... args) {
private synchronized void executeCommand(String query, Object... args) {
activateOnCurrentThread();
db.getTransaction().begin();
db.command(query, args);
db.getTransaction().commit();
Comment on lines -293 to +302
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why use synchronize here? Instead, you could catch OConcurrentModificationException and retry.

See: https://orientdb.org/docs/3.0.x/general/Concurrency.html

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. That could be done. But currently we have only this one Contentstore that is passed around everywhere, so all Threads use the same db session, which can lead to a deadlock especially when adding new documents.

I think It would be better to restructure the contentstore to have one single orient instance and each thread uses it's own database session. Like they recommend in the docs.

That's a bit more work that needs to be done. So I decided to keep it simple first and synchronize the potential critical parts.

}

public Set<String> getTags() {
Expand Down Expand Up @@ -353,6 +348,9 @@ private void createSignatureType(OSchema schema) {
OClass signatures = schema.createClass(Schema.SIGNATURES);
signatures.createProperty(ModelAttributes.SHA1, OType.STRING).setNotNull(true);
signatures.createIndex("sha1Idx", OClass.INDEX_TYPE.UNIQUE, ModelAttributes.SHA1);

signatures.createProperty("key", OType.STRING);
signatures.createIndex("kexIdx", OClass.INDEX_TYPE.UNIQUE, "key");
}

public void updateAndClearCacheIfNeeded(boolean needed, File templateFolder) {
Expand Down Expand Up @@ -407,10 +405,12 @@ public boolean isActive() {
return db.isActiveOnCurrentThread();
}

public void addDocument(DocumentModel document) {
public synchronized void addDocument(DocumentModel document) {
db.getTransaction().begin();
ODocument doc = new ODocument(Schema.DOCUMENTS);
doc.fromMap(document);
doc.save();
db.getTransaction().commit();
}

protected abstract class Schema {
Expand Down
Loading