From 7287a9ece6e7f498394314cac2bf506dfce64f60 Mon Sep 17 00:00:00 2001 From: sethgoldin <4632977+sethgoldin@users.noreply.github.com> Date: Mon, 9 Mar 2020 20:37:16 -0400 Subject: [PATCH 01/10] Few preliminary lines of code --- send.ps1 | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 send.ps1 diff --git a/send.ps1 b/send.ps1 new file mode 100644 index 0000000..96a67d6 --- /dev/null +++ b/send.ps1 @@ -0,0 +1,19 @@ +# Lets check to make sure that mhl is properly installed +if (Get-Command mhl -errorAction Stop) +{} + +# Lets check to make sure that aws is properly installed +if (Get-Command aws -errorAction Stop) +{} + +# Let's have the user specify which source folder should be uploaded into S3 + +$sourceLocalDirectory = Read-Host "From which directory on your local system are you uploading into AWS S3? Please enter the absolute path and escape any spaces if necessary: " + +# Now $sourceLocalDirectory will work as the variable for the source folder from the local system that will be ingested into S3 + +# Let's prompt the user for the name of the S3 bucket + +$s3BucketName = Read-Host "What should the name of the AWS S3 bucket be? The name of the bucket should take the form with only lowercase letters and hyphens, but should use NO uppercase letters nor spaces: " + +# Now $s3BucketName will work as the variable for the name of the S3 bucket \ No newline at end of file From 309aac8f763f61bee5dd080729603d7abaef5722 Mon Sep 17 00:00:00 2001 From: sethgoldin <4632977+sethgoldin@users.noreply.github.com> Date: Tue, 10 Mar 2020 10:43:42 -0400 Subject: [PATCH 02/10] Precompiled mhl.exe --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 35662be..48a6156 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,8 @@ The MHL file generated on the sending side and verified on the receiving side fu ## System requirements - The [MHL tool](https://github.com/pomfort/mhl-tool) should be installed into your `$PATH`. On CentOS 7.7 and Fedora 31, after compiling from source so that `mhl` will call the properly installed versions of the OpenSSL libraries, it is [recommended](https://unix.stackexchange.com/questions/8656/usr-bin-vs-usr-local-bin-on-linux/8658#8658) to manually move the `mhl` binary into `/usr/local/bin`, since the program will not be managed by the distribution's package manager. -- The [`.pkg` installer from Pomfort](http://download.pomfort.com/mhl-tool.zip) will install a precompiled binary for macOS into `/usr/local/bin`, which is included by default in macOS's `$PATH`. +- The [`.pkg` installer from Pomfort]() will install a precompiled binary for macOS into `/usr/local/bin`, which is included by default in macOS's `$PATH`. +- On Windows, download and extract [the precompiled binary from Pomfort](http://download.pomfort.com/mhl-tool.zip), and then copy or move `mhl.exe` into `C:\Windows\System32\`, which should already be included by default in the `Path` system environment variables. - The [AWS CLI](https://aws.amazon.com/cli/) should be installed and configured on both endpoints, with: - The sending IAM user having at least full S3 write access on the AWS account - The receiving IAM user having at least full S3 read access on the AWS account @@ -18,10 +19,11 @@ The MHL file generated on the sending side and verified on the receiving side fu - The command output format set to [text](https://docs.aws.amazon.com/cli/latest/userguide/cli-usage-output.html#text-output) ## Compatible platforms -Release 0.0.2 has tested on Linux and macOS endpoints, specifically on: +Release 0.0.3 has tested on Linux and macOS endpoints, specifically on: - Fedora 31 - CentOS 7.7 - macOS Catalina 10.15.3 +- Windows 10 1909 There aren't too many dependencies, so these scripts seem like they should work flawlessly on other major Linux distributions as well, though no other distributions have been tested. From 22dcd56325d9ec9d9658f562c771b3b290c9d5f5 Mon Sep 17 00:00:00 2001 From: sethgoldin <4632977+sethgoldin@users.noreply.github.com> Date: Tue, 10 Mar 2020 10:46:06 -0400 Subject: [PATCH 03/10] Fixed typo --- send.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/send.sh b/send.sh index 5b46cb7..1ae9826 100755 --- a/send.sh +++ b/send.sh @@ -33,7 +33,7 @@ cd "$sourceLocalDirectory" mhl seal -t xxhash64 * && -# We're using the 64-bit xxHash algorith specifically, because it's fast and reliable https://github.com/Cyan4973/xxHash +# We're using the 64-bit xxHash algorithm specifically, because it's fast and reliable https://github.com/Cyan4973/xxHash # Now that we've sealed the contents of the folder, let's sync the data from the local folder into the bucket https://docs.aws.amazon.com/cli/latest/reference/s3/sync.html From e86895c64a9ca7e57d47f7cfd19d1bba4834a7f6 Mon Sep 17 00:00:00 2001 From: sethgoldin <4632977+sethgoldin@users.noreply.github.com> Date: Tue, 10 Mar 2020 11:08:31 -0400 Subject: [PATCH 04/10] send.ps1 first draft Seems to be working OK on Windows 10 --- send.ps1 | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/send.ps1 b/send.ps1 index 96a67d6..94ad77c 100644 --- a/send.ps1 +++ b/send.ps1 @@ -1,4 +1,4 @@ -# Lets check to make sure that mhl is properly installed +# Lets check to make sure that mhl is properly installed if (Get-Command mhl -errorAction Stop) {} @@ -8,12 +8,33 @@ if (Get-Command aws -errorAction Stop) # Let's have the user specify which source folder should be uploaded into S3 -$sourceLocalDirectory = Read-Host "From which directory on your local system are you uploading into AWS S3? Please enter the absolute path and escape any spaces if necessary: " +$sourceLocalDirectory = Read-Host "From which directory on your local system are you uploading into AWS S3? Please enter the absolute path and escape any spaces if necessary" # Now $sourceLocalDirectory will work as the variable for the source folder from the local system that will be ingested into S3 # Let's prompt the user for the name of the S3 bucket -$s3BucketName = Read-Host "What should the name of the AWS S3 bucket be? The name of the bucket should take the form with only lowercase letters and hyphens, but should use NO uppercase letters nor spaces: " +$s3BucketName = Read-Host "What should the name of the AWS S3 bucket be? The name of the bucket should take the form with only lowercase letters and hyphens, but should use NO uppercase letters nor spaces" -# Now $s3BucketName will work as the variable for the name of the S3 bucket \ No newline at end of file +# Now $s3BucketName will work as the variable for the name of the S3 bucket + +# Let's make the S3 bucket first, because this is probably the most likely source of an error. According to Amazon, "An Amazon S3 bucket name is globally unique, and the namespace is shared by all AWS accounts. This means that after a bucket is created, the name of that bucket cannot be used by another AWS account in any AWS Region until the bucket is deleted. You should not depend on specific bucket naming conventions for availability or security verification purposes." + +aws s3 mb $s3BucketName; + +# Let's cd into the source directory, and then execute mhl seal for the whole directory, with the xxHash algorithm, which is nice and fast +# N.B. mhl must be run from inside the source directory, so best practice is to cd in to the directory right within the shell script itself: https://stackoverflow.com/a/10566581/ + +cd $sourceLocalDirectory; + +mhl seal -t xxhash64 *; + +# We're using the 64-bit xxHash algorithm specifically, because it's fast and reliable https://github.com/Cyan4973/xxHash + +# Now that we've sealed the contents of the folder, let's sync the data from the local folder into the bucket https://docs.aws.amazon.com/cli/latest/reference/s3/sync.html# Now that we've sealed the contents of the folder, let's sync the data from the local folder into the bucket https://docs.aws.amazon.com/cli/latest/reference/s3/sync.html# Now that we've sealed the contents of the folder, let's sync the data from the local folder into the bucket https://docs.aws.amazon.com/cli/latest/reference/s3/sync.html + +aws s3 sync "$sourceLocalDirectory" $s3BucketName; + +# Once the upload has finished, let's let the user know that the data has been sealed and ingested. + +echo "The data from <$sourceLocalDirectory> has been sealed with xxHash checksums and has been ingested into the AWS S3 bucket named <$s3BucketName>." From 293f022535ea9b5c58cffe392891ff7438b3f66f Mon Sep 17 00:00:00 2001 From: sethgoldin <4632977+sethgoldin@users.noreply.github.com> Date: Tue, 10 Mar 2020 16:22:40 -0400 Subject: [PATCH 05/10] Add files via upload --- receive.ps1 | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 receive.ps1 diff --git a/receive.ps1 b/receive.ps1 new file mode 100644 index 0000000..514d036 --- /dev/null +++ b/receive.ps1 @@ -0,0 +1,62 @@ +# Lets check to make sure that mhl is properly installed +if (Get-Command mhl -errorAction Stop) +{} + +# Lets check to make sure that aws is properly installed +if (Get-Command aws -errorAction Stop) +{} + +# Let's have the user specify from which bucket they'll be downloading + +$s3BucketName = Read-Host "What is the name of the AWS S3 bucket from which you'll be downloading the data? The name of a bucket takes the form with only lowercase letters and hyphens, but uses NO uppercase letters nor spaces" + +# Let's have the user specify exactly into which directory on the local system they want the data to go + +Function Get-Folder($initialDirectory) + +{ + [System.Reflection.Assembly]::LoadWithPartialName("System.windows.forms")|Out-Null + + $foldername = New-Object System.Windows.Forms.FolderBrowserDialog + $foldername.Description = "Select a folder" + $foldername.rootfolder = "MyComputer" + + if($foldername.ShowDialog() -eq "OK") + { + $folder += $foldername.SelectedPath + } + return $folder +} + +echo "Into which local directory on your system are you downloading the data?" + +$destinationLocalDirectory = Get-Folder + +# Now $destinationLocalDirectory will work as the variable for the destination folder on the local system into which the data will go + +# Let's now sync from the S3 bucket to the local system https://docs.aws.amazon.com/cli/latest/reference/s3/sync.html + +aws s3 sync $s3BucketName $destinationLocalDirectory; + +# Let's check to make sure that a .mhl file exists in the destination. + +cd $destinationLocalDirectory + +# If there are no MHL files in this directory, we'll throw an error. + +if (((Get-ChildItem -Path $destinationLocalDirectory -filter *.mhl | Measure-Object | Select-Object -expandProperty Count) -eq 0)) + {echo "ERROR: The local directory does not seem to have an MHL file with which to verify the contents. The data integrity of the contents of this directory cannot be verified."; Exit-PSSession} + +# If there are more than one MHL files in the directory, we'll throw an error, because we don't know which MHL file to check. + +elseif (((Get-ChildItem -Path $destinationLocalDirectory -filter *.mhl | Measure-Object | Select-Object -expandProperty Count) -gt 1)) + { echo "ERROR: There are more than one MHL files in the directory, so this script does not know which MHL to use to verify the contents of the directory. The data integrity of the contents of this directory cannot be verified." ; Exit-PSSession} + +# If there's exactly one MHL file, let's grab the name of it and store that into a variable, and then verify the MHL file we found. Once the download has finished and the MHL file has been verified, let's let the user know that the data has been downloaded and verified. + +else + { $mhlFileName = gci *.mhl; mhl verify -f $mhlFileName; + if ($LASTEXITCODE -ne 0) + { Exit-PSSession } + else + { echo "The data from the AWS S3 bucket named <$s3BucketName> has been downloaded into $destinationLocalDirectory and has been verified." }} \ No newline at end of file From 6dd4b8134e100ad213bc95893dc97766bc010c09 Mon Sep 17 00:00:00 2001 From: sethgoldin <4632977+sethgoldin@users.noreply.github.com> Date: Tue, 10 Mar 2020 16:41:42 -0400 Subject: [PATCH 06/10] Using GUI to prompt for source folder --- send.ps1 | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/send.ps1 b/send.ps1 index 94ad77c..d2e3dfe 100644 --- a/send.ps1 +++ b/send.ps1 @@ -8,7 +8,25 @@ if (Get-Command aws -errorAction Stop) # Let's have the user specify which source folder should be uploaded into S3 -$sourceLocalDirectory = Read-Host "From which directory on your local system are you uploading into AWS S3? Please enter the absolute path and escape any spaces if necessary" +Function Get-Folder($initialDirectory) + +{ + [System.Reflection.Assembly]::LoadWithPartialName("System.windows.forms")|Out-Null + + $foldername = New-Object System.Windows.Forms.FolderBrowserDialog + $foldername.Description = "Select a folder" + $foldername.rootfolder = "MyComputer" + + if($foldername.ShowDialog() -eq "OK") + { + $folder += $foldername.SelectedPath + } + return $folder +} + +echo "From which directory on your local system are you uploading into AWS S3?" + +$sourceLocalDirectory = Get-Folder # Now $sourceLocalDirectory will work as the variable for the source folder from the local system that will be ingested into S3 @@ -22,19 +40,31 @@ $s3BucketName = Read-Host "What should the name of the AWS S3 bucket be? The nam aws s3 mb $s3BucketName; +if ($LASTEXITCODE -ne 0) + { Exit-PSSession } + # Let's cd into the source directory, and then execute mhl seal for the whole directory, with the xxHash algorithm, which is nice and fast # N.B. mhl must be run from inside the source directory, so best practice is to cd in to the directory right within the shell script itself: https://stackoverflow.com/a/10566581/ cd $sourceLocalDirectory; +if ($LASTEXITCODE -ne 0) + { Exit-PSSession } + mhl seal -t xxhash64 *; +if ($LASTEXITCODE -ne 0) + { Exit-PSSession } + # We're using the 64-bit xxHash algorithm specifically, because it's fast and reliable https://github.com/Cyan4973/xxHash # Now that we've sealed the contents of the folder, let's sync the data from the local folder into the bucket https://docs.aws.amazon.com/cli/latest/reference/s3/sync.html# Now that we've sealed the contents of the folder, let's sync the data from the local folder into the bucket https://docs.aws.amazon.com/cli/latest/reference/s3/sync.html# Now that we've sealed the contents of the folder, let's sync the data from the local folder into the bucket https://docs.aws.amazon.com/cli/latest/reference/s3/sync.html aws s3 sync "$sourceLocalDirectory" $s3BucketName; +if ($LASTEXITCODE -ne 0) + { Exit-PSSession } + # Once the upload has finished, let's let the user know that the data has been sealed and ingested. echo "The data from <$sourceLocalDirectory> has been sealed with xxHash checksums and has been ingested into the AWS S3 bucket named <$s3BucketName>." From dee85356b730297dab0c8b759b18a7985d1de90c Mon Sep 17 00:00:00 2001 From: sethgoldin <4632977+sethgoldin@users.noreply.github.com> Date: Tue, 10 Mar 2020 17:06:33 -0400 Subject: [PATCH 07/10] Update README.md --- README.md | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 48a6156..d903662 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,33 @@ # s3-xxHash-transfer -Included in this repository are a couple of shell scripts that glue together the [MHL tool](https://github.com/pomfort/mhl-tool) and the [AWS CLI](https://docs.aws.amazon.com/cli/index.html). This allows for a workflow that can transfer enormous amounts of data through an [S3 bucket](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html) with extremely fast checksum verification. These scripts ensure bulletproof data integrity, verifying every single bit, with blazingly fast speed afforded by 64-bit xxHash and AWS S3. +Included in this repository are shell scripts that glue together the [MHL tool](https://github.com/pomfort/mhl-tool) and the [AWS CLI](https://docs.aws.amazon.com/cli/index.html). This allows for a workflow that can transfer enormous amounts of data through an [S3 bucket](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html) with extremely fast checksum verification. These scripts ensure bulletproof data integrity, verifying every single bit, with blazingly fast speed afforded by 64-bit xxHash and AWS S3. ## Workflow -1. On the "source" computer, run `$ sh send.sh`. `sudo` privilege is _not_ required. `send.sh` will prompt for the source destination directory, "seal" the contents of the directory with 64-bit xxHash checksums, prompt for the name of a new S3 bucket, automatically make that bucket, and then ingest the entire directory into the bucket. -2. On the "destination" computer, run `$ sh receive.sh`. `sudo` privilege is _not_ required. `receive.sh` will prompt for the name of the S3 bucket that had been created by `send.sh`, prompt for the local directory path into where the data will be downloaded, and then will automatically download all data from the S3 bucket and verify the 64-bit xxHash checksums for every single file. +1. On the "source" computer: + 1. Depending on your OS: + 1. On macOS or Linux, run `$ sh send.sh`. `sudo` privilege is _not_ required. + 1. On Windows, run `send.ps1` via PowerShell. + 1. The "send" script will prompt for the source destination directory, "seal" the contents of the directory with 64-bit xxHash checksums, prompt for the name of a new S3 bucket, automatically make that bucket, and then ingest the entire directory into the bucket. +1. On the "destination" computer: + 1. Depending on your OS: + 1. On macOS or Linux, run `$ sh receive.sh`. `sudo` privilege is _not_ required. + 1. On Windows, run `receive.ps1` via PowerShell. + 1. The "receive" script will prompt for the name of the S3 bucket that had been created by the "send" script, prompt for the local directory path into where the data will be downloaded, and then will automatically download all data from the S3 bucket and verify the 64-bit xxHash checksums for every single file. The MHL file generated on the sending side and verified on the receiving side functions as as a kind of manifest for the data, which ensures end-to-end data integrity. These scripts use the extremely fast [64-bit xxHash hashing algorithm](https://github.com/Cyan4973/xxHash). ## System requirements -- The [MHL tool](https://github.com/pomfort/mhl-tool) should be installed into your `$PATH`. On CentOS 7.7 and Fedora 31, after compiling from source so that `mhl` will call the properly installed versions of the OpenSSL libraries, it is [recommended](https://unix.stackexchange.com/questions/8656/usr-bin-vs-usr-local-bin-on-linux/8658#8658) to manually move the `mhl` binary into `/usr/local/bin`, since the program will not be managed by the distribution's package manager. +- The [MHL tool](https://github.com/pomfort/mhl-tool) should be installed into your `$PATH`. On CentOS 7.7 and Fedora 31, after compiling from source so that `mhl` will call the properly installed versions of the OpenSSL libraries, it is [recommended](https://unix.stackexchange.com/questions/8656/usr-bin-vs-usr-local-bin-on-linux/8658#8658) that you manually move the `mhl` binary into `/usr/local/bin`, since the program will not be managed by the distribution's package manager. - The [`.pkg` installer from Pomfort]() will install a precompiled binary for macOS into `/usr/local/bin`, which is included by default in macOS's `$PATH`. -- On Windows, download and extract [the precompiled binary from Pomfort](http://download.pomfort.com/mhl-tool.zip), and then copy or move `mhl.exe` into `C:\Windows\System32\`, which should already be included by default in the `Path` system environment variables. +- On Windows, download and extract [the precompiled binary from Pomfort](http://download.pomfort.com/mhl-tool.zip), and then copy or move `mhl.exe` into `C:\Windows\System32\`, which is included by default in the Windows `Path` system environment variables. - The [AWS CLI](https://aws.amazon.com/cli/) should be installed and configured on both endpoints, with: - The sending IAM user having at least full S3 write access on the AWS account - The receiving IAM user having at least full S3 read access on the AWS account - Both endpoints connected to the same [region](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) - The command output format set to [text](https://docs.aws.amazon.com/cli/latest/userguide/cli-usage-output.html#text-output) -## Compatible platforms -Release 0.0.3 has tested on Linux and macOS endpoints, specifically on: +## Tested platforms +Release 0.0.3 has tested on Linux, macOS, and Windows, specifically on: - Fedora 31 - CentOS 7.7 - macOS Catalina 10.15.3 @@ -27,8 +35,4 @@ Release 0.0.3 has tested on Linux and macOS endpoints, specifically on: There aren't too many dependencies, so these scripts seem like they should work flawlessly on other major Linux distributions as well, though no other distributions have been tested. -Both the MHL tool and the AWS CLI are available across Linux, macOS, and Windows, so the same `bash` scripts work identically on Linux and macOS. - Though `zsh` is now the default shell on macOS Catalina, the script runs in `bash`, as specified from the first line of the script: `#!/bin/bash`. For now, Catalina still ships `bash`. Whether future releases of macOS will contain `bash` is an open question. The scripts may need to be modified in the future to run natively in `zsh`, but at least for now, on Catalina, `bash` works. - -The [Windows command shells](https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/windows-commands), both the Command shell and PowerShell, are quite a bit different than `bash`, so porting to Windows will take a bit more effort, and will come in a future release. From 2a41ce464e3f5dc2bfb93d6ad13565a3c2b32423 Mon Sep 17 00:00:00 2001 From: sethgoldin <4632977+sethgoldin@users.noreply.github.com> Date: Tue, 10 Mar 2020 17:09:11 -0400 Subject: [PATCH 08/10] Update README.md --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d903662..148e60b 100644 --- a/README.md +++ b/README.md @@ -4,15 +4,15 @@ Included in this repository are shell scripts that glue together the [MHL tool]( ## Workflow 1. On the "source" computer: - 1. Depending on your OS: - 1. On macOS or Linux, run `$ sh send.sh`. `sudo` privilege is _not_ required. - 1. On Windows, run `send.ps1` via PowerShell. - 1. The "send" script will prompt for the source destination directory, "seal" the contents of the directory with 64-bit xxHash checksums, prompt for the name of a new S3 bucket, automatically make that bucket, and then ingest the entire directory into the bucket. + 1. Depending on your OS: + 1. On macOS or Linux, run `$ sh send.sh`. `sudo` privilege is _not_ required. + 1. On Windows, run `send.ps1` via PowerShell. + 1. The "send" script will prompt for the source destination directory, "seal" the contents of the directory with 64-bit xxHash checksums, prompt for the name of a new S3 bucket, automatically make that bucket, and then ingest the entire directory into the bucket. 1. On the "destination" computer: - 1. Depending on your OS: - 1. On macOS or Linux, run `$ sh receive.sh`. `sudo` privilege is _not_ required. - 1. On Windows, run `receive.ps1` via PowerShell. - 1. The "receive" script will prompt for the name of the S3 bucket that had been created by the "send" script, prompt for the local directory path into where the data will be downloaded, and then will automatically download all data from the S3 bucket and verify the 64-bit xxHash checksums for every single file. + 1. Depending on your OS: + 1. On macOS or Linux, run `$ sh receive.sh`. `sudo` privilege is _not_ required. + 1. On Windows, run `receive.ps1` via PowerShell. + 1. The "receive" script will prompt for the name of the S3 bucket that had been created by the "send" script, prompt for the local directory path into where the data will be downloaded, and then will automatically download all data from the S3 bucket and verify the 64-bit xxHash checksums for every single file. The MHL file generated on the sending side and verified on the receiving side functions as as a kind of manifest for the data, which ensures end-to-end data integrity. These scripts use the extremely fast [64-bit xxHash hashing algorithm](https://github.com/Cyan4973/xxHash). From 38217352aba14fa12862bc37c2b690aa010686b0 Mon Sep 17 00:00:00 2001 From: sethgoldin <4632977+sethgoldin@users.noreply.github.com> Date: Tue, 10 Mar 2020 17:13:15 -0400 Subject: [PATCH 09/10] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 148e60b..2210c00 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,12 @@ Included in this repository are shell scripts that glue together the [MHL tool]( 1. On the "source" computer: 1. Depending on your OS: 1. On macOS or Linux, run `$ sh send.sh`. `sudo` privilege is _not_ required. - 1. On Windows, run `send.ps1` via PowerShell. + 1. On Windows, run `PS> send.ps1`. 1. The "send" script will prompt for the source destination directory, "seal" the contents of the directory with 64-bit xxHash checksums, prompt for the name of a new S3 bucket, automatically make that bucket, and then ingest the entire directory into the bucket. 1. On the "destination" computer: 1. Depending on your OS: 1. On macOS or Linux, run `$ sh receive.sh`. `sudo` privilege is _not_ required. - 1. On Windows, run `receive.ps1` via PowerShell. + 1. On Windows, run `PS> receive.ps1`. 1. The "receive" script will prompt for the name of the S3 bucket that had been created by the "send" script, prompt for the local directory path into where the data will be downloaded, and then will automatically download all data from the S3 bucket and verify the 64-bit xxHash checksums for every single file. The MHL file generated on the sending side and verified on the receiving side functions as as a kind of manifest for the data, which ensures end-to-end data integrity. These scripts use the extremely fast [64-bit xxHash hashing algorithm](https://github.com/Cyan4973/xxHash). From e9181b14505451b8840cf5295f0e8d9713160b00 Mon Sep 17 00:00:00 2001 From: sethgoldin <4632977+sethgoldin@users.noreply.github.com> Date: Tue, 10 Mar 2020 17:19:54 -0400 Subject: [PATCH 10/10] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2210c00..d4cc379 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ The MHL file generated on the sending side and verified on the receiving side fu ## System requirements - The [MHL tool](https://github.com/pomfort/mhl-tool) should be installed into your `$PATH`. On CentOS 7.7 and Fedora 31, after compiling from source so that `mhl` will call the properly installed versions of the OpenSSL libraries, it is [recommended](https://unix.stackexchange.com/questions/8656/usr-bin-vs-usr-local-bin-on-linux/8658#8658) that you manually move the `mhl` binary into `/usr/local/bin`, since the program will not be managed by the distribution's package manager. -- The [`.pkg` installer from Pomfort]() will install a precompiled binary for macOS into `/usr/local/bin`, which is included by default in macOS's `$PATH`. +- The [`.pkg` installer from Pomfort](http://download.pomfort.com/mhl-tool.zip) will install a precompiled binary for macOS into `/usr/local/bin`, which is included by default in macOS's `$PATH`. - On Windows, download and extract [the precompiled binary from Pomfort](http://download.pomfort.com/mhl-tool.zip), and then copy or move `mhl.exe` into `C:\Windows\System32\`, which is included by default in the Windows `Path` system environment variables. - The [AWS CLI](https://aws.amazon.com/cli/) should be installed and configured on both endpoints, with: - The sending IAM user having at least full S3 write access on the AWS account