NIFIREG-260 Rebase capability to support branching workflow

nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/service/alias/RegistryUrlAliasService.java

@@ -94,7 +94,7 @@ private static List<Alias> createAliases(NiFiRegistryProperties niFiRegistryProp
         }
     }
 
-    protected RegistryUrlAliasService(List<Alias> aliases) {
+    public RegistryUrlAliasService(List<Alias> aliases) {
         Pattern urlStart = Pattern.compile("^https?://");
 
         this.aliases = new LinkedHashMap<>();
@@ -137,7 +137,7 @@ public void setInternal(VersionedProcessGroup processGroup) {
         }
     }
 
-    protected String getExternal(String url) {
+    public String getExternal(String url) {
         for (Map.Entry<String, String> alias : aliases.entrySet()) {
             String internal = alias.getKey();
             String external = alias.getValue();
@@ -153,7 +153,7 @@ protected String getExternal(String url) {
         return url;
     }
 
-    protected String getInternal(String url) {
+    public String getInternal(String url) {
         for (Map.Entry<String, String> alias : aliases.entrySet()) {
             String internal = alias.getKey();
             String external = alias.getValue();

nifi-registry-toolkit/README.md

@@ -16,8 +16,6 @@
 
 This submodule is a landing zone for command line utilities that can be used for maintenance/automation of registry actions.
 
-It currently only contains a migration tool for changing flow persistence providers.
-
 ## Build
 
 ```
@@ -32,4 +30,29 @@ mvn clean install
 1. Edit providers-to.xml to reflect what you'd like to migrate to (e.g. git)
 1. In registry home as working directory, run persistence-toolkit.sh -t providers-to.xml
 1. Rename providers-to.xml -> providers.xml
-1. Start registry back up
+1. Start registry back up
+
+
+## Registry rebase (EXPERIMENTAL)
+
+Initial support for rebasing a branched registry's changes on top of the upstream.
+
+### Workflow:
+
+#### Generate diff:
+1. "Branch" the upstream registry by shutting it down, copying directory to new location.
+1. Start upstream registry back up
+1. Configure branched registry with new hostname, registry-aliases.xml to account for new url
+1. Point NiFi at the branched registry, make some changes, decide you want to push them upstream
+1. Configure conf/branch-nifi-registry-client.properties and conf/upstream-nifi-registry-client.properties to point at the branched, upstream registries
+1. Generate a rebase diff
+```
+bin/rebase-toolkit.sh diff --upstreamPropertiesFile conf/upstream-nifi-registry-client.properties --branchPropertiesFile conf/branch-nifi-registry-client.properties --bucketId BUCKET_UUID --flowId FLOW_UUID --output output.yaml
+```
+
+#### Apply upstream:
+1. Review output.yaml, opportunity for peer review as well
+1. Apply rebase diff
+```
+bin/rebase-toolkit.sh apply --upstreamPropertiesFile conf/upstream-nifi-registry-client.properties --branchPropertiesFile conf/branch-nifi-registry-client.properties --input output.yaml
+```

nifi-registry-toolkit/nifi-registry-toolkit-assembly/pom.xml

@@ -55,6 +55,11 @@ language governing permissions and limitations under the License. -->
             <artifactId>nifi-registry-toolkit-persistence</artifactId>
             <version>0.4.0-SNAPSHOT</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.nifi.registry</groupId>
+            <artifactId>nifi-registry-toolkit-rebase</artifactId>
+            <version>0.4.0-SNAPSHOT</version>
+        </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>

nifi-registry-toolkit/nifi-registry-toolkit-assembly/src/main/assembly/dependencies.xml

@@ -45,6 +45,11 @@
             <outputDirectory>classpath/</outputDirectory>
             <fileMode>0600</fileMode>
         </fileSet>
+        <fileSet>
+            <directory>${project.basedir}/src/main/resources/conf</directory>
+            <outputDirectory>conf/</outputDirectory>
+            <fileMode>0600</fileMode>
+        </fileSet>
     </fileSets>
     <files>
         <file>

nifi-registry-toolkit/nifi-registry-toolkit-assembly/src/main/resources/bin/rebase-toolkit.bat

@@ -0,0 +1,41 @@
+@echo off
+rem
+rem    Licensed to the Apache Software Foundation (ASF) under one or more
+rem    contributor license agreements.  See the NOTICE file distributed with
+rem    this work for additional information regarding copyright ownership.
+rem    The ASF licenses this file to You under the Apache License, Version 2.0
+rem    (the "License"); you may not use this file except in compliance with
+rem    the License.  You may obtain a copy of the License at
+rem
+rem       http://www.apache.org/licenses/LICENSE-2.0
+rem
+rem    Unless required by applicable law or agreed to in writing, software
+rem    distributed under the License is distributed on an "AS IS" BASIS,
+rem    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+rem    See the License for the specific language governing permissions and
+rem    limitations under the License.
+rem
+
+rem Use JAVA_HOME if it's set; otherwise, just use java
+
+if "%JAVA_HOME%" == "" goto noJavaHome
+if not exist "%JAVA_HOME%\bin\java.exe" goto noJavaHome
+set JAVA_EXE=%JAVA_HOME%\bin\java.exe
+goto startConfig
+
+:noJavaHome
+echo The JAVA_HOME environment variable is not defined correctly. 1>&2
+echo Instead the PATH will be used to find the java executable. 1>&2
+echo. 1>&2
+set JAVA_EXE=java
+goto startConfig
+
+:startConfig
+set LIB_DIR=%~dp0..\classpath;%~dp0..\lib
+
+if "%JAVA_OPTS%" == "" set JAVA_OPTS=-Xms12m -Xmx24m
+
+SET JAVA_PARAMS=-cp %LIB_DIR%\* %JAVA_OPTS% org.apache.nifi.registry.toolkit.rebase.RecursiveRebase
+
+cmd.exe /C ""%JAVA_EXE%" %JAVA_PARAMS% %* ""
+

nifi-registry-toolkit/nifi-registry-toolkit-assembly/src/main/resources/bin/rebase-toolkit.sh

@@ -0,0 +1,120 @@
+#!/bin/sh
+#
+#    Licensed to the Apache Software Foundation (ASF) under one or more
+#    contributor license agreements.  See the NOTICE file distributed with
+#    this work for additional information regarding copyright ownership.
+#    The ASF licenses this file to You under the Apache License, Version 2.0
+#    (the "License"); you may not use this file except in compliance with
+#    the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+#
+
+# Script structure inspired from Apache Karaf and other Apache projects with similar startup approaches
+
+SCRIPT_DIR=$(dirname "$0")
+SCRIPT_NAME=$(basename "$0")
+NIFI_REGISTRY_TOOLKIT_HOME=$(cd "${SCRIPT_DIR}" && cd .. && pwd)
+PROGNAME=$(basename "$0")
+
+
+warn() {
+    echo "${PROGNAME}: $*" > /dev/stderr
+}
+
+die() {
+    warn "$*"
+    exit 1
+}
+
+detectOS() {
+    # OS specific support (must be 'true' or 'false').
+    cygwin=false;
+    aix=false;
+    os400=false;
+    darwin=false;
+    case "$(uname)" in
+        CYGWIN*)
+            cygwin=true
+            ;;
+        AIX*)
+            aix=true
+            ;;
+        OS400*)
+            os400=true
+            ;;
+        Darwin)
+            darwin=true
+            ;;
+    esac
+    # For AIX, set an environment variable
+    if ${aix}; then
+         export LDR_CNTRL=MAXDATA=0xB0000000@DSA
+         echo ${LDR_CNTRL} > /dev/stderr
+    fi
+}
+
+locateJava() {
+    # Setup the Java Virtual Machine
+    if $cygwin ; then
+        [ -n "${JAVA}" ] && JAVA=$(cygpath --unix "${JAVA}")
+        [ -n "${JAVA_HOME}" ] && JAVA_HOME=$(cygpath --unix "${JAVA_HOME}")
+    fi
+
+    if [ "x${JAVA}" = "x" ] && [ -r /etc/gentoo-release ] ; then
+        JAVA_HOME=$(java-config --jre-home)
+    fi
+    if [ "x${JAVA}" = "x" ]; then
+        if [ "x${JAVA_HOME}" != "x" ]; then
+            if [ ! -d "${JAVA_HOME}" ]; then
+                die "JAVA_HOME is not valid: ${JAVA_HOME}"
+            fi
+            JAVA="${JAVA_HOME}/bin/java"
+        else
+            warn "JAVA_HOME not set; results may vary"
+            JAVA=$(type java)
+            JAVA=$(expr "${JAVA}" : '.* \(/.*\)$')
+            if [ "x${JAVA}" = "x" ]; then
+                die "java command not found"
+            fi
+        fi
+    fi
+}
+
+init() {
+    # Determine if there is special OS handling we must perform
+    detectOS
+
+    # Locate the Java VM to execute
+    locateJava "$1"
+}
+
+run() {
+    LIBS="${NIFI_REGISTRY_TOOLKIT_HOME}/lib/*"
+
+    sudo_cmd_prefix=""
+    if $cygwin; then
+        NIFI_REGISTRY_TOOLKIT_HOME=$(cygpath --path --windows "${NIFI_REGISTRY_TOOLKIT_HOME}")
+        CLASSPATH="$NIFI_REGISTRY_TOOLKIT_HOME/classpath;$(cygpath --path --windows "${LIBS}")"
+    else
+        CLASSPATH="$NIFI_REGISTRY_TOOLKIT_HOME/classpath:${LIBS}"
+    fi
+
+   export JAVA_HOME="$JAVA_HOME"
+   export NIFI_REGISTRY_TOOLKIT_HOME="$NIFI_REGISTRY_TOOLKIT_HOME"
+
+   umask 0077
+   echo "$CLASSPATH" > /dev/stderr
+   exec "${JAVA}" -cp "${CLASSPATH}" ${JAVA_OPTS:--Xms12m -Xmx256m} org.apache.nifi.registry.toolkit.rebase.RecursiveRebase "$@"
+}
+
+
+init "$1"
+run "$@"

nifi-registry-toolkit/nifi-registry-toolkit-assembly/src/main/resources/conf/branch-nifi-registry-client.properties

@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# nifi.registry.web.http.host=http://branch-registry.local:18080
+# nifi.registry.web.https.host=https://branch-registry.local:18443
+
+# nifi.registry.external.url=http://branch-registry.local:18080
+
+# client security properties #
+# nifi.registry.security.keystore=./conf/branch-keystore.jks
+# nifi.registry.security.keystoreType=JKS
+# nifi.registry.security.keystorePasswd=password
+# nifi.registry.security.keyPasswd=password
+# nifi.registry.security.truststore=./conf/branch-truststore.jks
+# nifi.registry.security.truststoreType=JKS
+# nifi.registry.security.truststorePasswd=password
+# nifi.registry.client.read.timeout=60 s
+# nifi.registry.client.connect.timeout=60 s

nifi-registry-toolkit/nifi-registry-toolkit-assembly/src/main/resources/conf/upstream-nifi-registry-client.properties

@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# nifi.registry.web.http.host=http://upstream-registry.local:18080
+# nifi.registry.web.https.host=https://upstream-registry.local:18443
+
+# nifi.registry.external.url=http://upstream-registry.local:18080
+
+# client security properties #
+# nifi.registry.security.keystore=./conf/upstream-keystore.jks
+# nifi.registry.security.keystoreType=JKS
+# nifi.registry.security.keystorePasswd=password
+# nifi.registry.security.keyPasswd=password
+# nifi.registry.security.truststore=./conf/upstream-truststore.jks
+# nifi.registry.security.truststoreType=JKS
+# nifi.registry.security.truststorePasswd=password
+# nifi.registry.client.read.timeout=60 s
+# nifi.registry.client.connect.timeout=60 s