diff --git a/.gitignore b/.gitignore index 4c0c6bb..20b409b 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,5 @@ terraform/SAVE.mysettings.tfvars dev.tfvars my.tfvars my.plan +build/ +cloudflow/build/ diff --git a/TODO b/TODO new file mode 100644 index 0000000..0432480 --- /dev/null +++ b/TODO @@ -0,0 +1,10 @@ +add ulimit -s unlimited to every user profile + +commit changes +test other builds +clean up +commit and tag as 1.5.0b +commit and tag nosofs as ioos.1.5.0b + +Test liveocean build and run +commit and tag liveocean as ioos.1.5.0b diff --git a/cloudflow/cluster/AWSCluster.py b/cloudflow/cluster/AWSCluster.py index d0e3356..8bb1040 100644 --- a/cloudflow/cluster/AWSCluster.py +++ b/cloudflow/cluster/AWSCluster.py @@ -33,6 +33,8 @@ formatter = logging.Formatter(' %(asctime)s %(levelname)s | %(message)s') fh.setFormatter(formatter) +efatypes=['c5', 'hpc' ] + # To avoid duplicate entries, only have one handler # This log might have a handler in one of their higher level scripts # This didn't work - still got duplicates, even when also added in main caller @@ -345,7 +347,8 @@ def getHosts(self): hosts = [] for instance in self.__instances: - hosts.append(instance.private_dns_name) + # hosts.append(instance.private_dns_name) + hosts.append(instance.private_ip_address) return hosts @@ -364,7 +367,7 @@ def getHostsCSV(self): cnt = 0 for instance in self.__instances: cnt += 1 - hostname = instance.private_dns_name + hostname = instance.private_ip_address # no comma on last host if cnt == instcnt: hosts += hostname @@ -375,12 +378,15 @@ def getHostsCSV(self): def __placementGroup(self): - """ This is a bit of a hack to satisfy AWS. Only c5 and c5n type of instances support placement group """ + """ This is a bit of a hack to satisfy AWS. Only some instances support placement group """ group = {} - if self.nodeType.startswith('c5'): + + + if self.nodeType.startswith(tuple(efatypes)): group = {'GroupName': self.placement_group} + return group @@ -393,16 +399,17 @@ def __netInterface(self): Also attaches security groups """ interface = { - 'AssociatePublicIpAddress': True, + 'AssociatePublicIpAddress': False, 'DeleteOnTermination': True, - 'Description': 'Network adaptor via boto3 api', + 'Description': 'Network adaptor via cloudflow boto3 api', 'DeviceIndex': 0, 'Groups': self.sg_ids, 'SubnetId': self.subnet_id } # if self.nodeType == 'c5n.18xlarge': - if self.nodeType in nodeInfo.efaTypes: + + if self.nodeType.startswith(tuple(efatypes)): interface['InterfaceType'] = 'efa' return interface diff --git a/cloudflow/cluster/configs/adnoc.config b/cloudflow/cluster/configs/adnoc.config deleted file mode 100644 index 3e7e2d6..0000000 --- a/cloudflow/cluster/configs/adnoc.config +++ /dev/null @@ -1,16 +0,0 @@ -{ -"platform" : "AWS", -"region" : "us-east-1", -"nodeType" : "c5.9xlarge", -"nodeCount" : 1, -"tags" : [ - { "Key": "Name", "Value": "ADNOC-forecast" }, - { "Key": "NAME", "Value": "adnoc-fcst" }, - { "Key": "Project", "Value": "ADNOC" } - ], -"image_id" : "ami-01f7b31e6a631b859", -"key_name" : "forecasting", -"sg_ids" : ["sg-073fcfb0ab31a1d62","sg-0b82904b0c646072d"], -"subnet_id" : "subnet-f19221a8", -"placement_group" : "ADNOC-cluster-placement" -} diff --git a/cloudflow/cluster/configs/debug.config b/cloudflow/cluster/configs/debug.config deleted file mode 100644 index 9c7cb69..0000000 --- a/cloudflow/cluster/configs/debug.config +++ /dev/null @@ -1,16 +0,0 @@ -{ -"platform" : "AWS", -"region" : "us-east-1", -"nodeType" : "t3.large", -"nodeCount" : 1, -"tags" : [ - { "Key": "Name", "Value": "IOOS-cloud-sandbox" }, - { "Key": "Project", "Value": "IOOS-cloud-sandbox" }, - { "Key": "NAME", "Value": "nsofs-fcst" } - ], -"image_id" : "ami-052f3e9220b9961dc", -"key_name" : "patrick-ioos-cloud-sandbox", -"sg_ids" : ["sg-006041073bfa7b072", "sg-0a48755f7b926b051", "sg-04a6bcecaec589f64"], -"subnet_id" : "subnet-09dae53e246bd68e4", -"placement_group" : "IOOS-cloud-sandbox-cluster-placement-group" -} diff --git a/cloudflow/cluster/configs/ioos.amd.config b/cloudflow/cluster/configs/ioos.amd.config index cf2f342..1dd7e66 100644 --- a/cloudflow/cluster/configs/ioos.amd.config +++ b/cloudflow/cluster/configs/ioos.amd.config @@ -2,19 +2,19 @@ "platform" : "AWS", "region" : "us-east-2", "nodeType" : "hpc6a.48xlarge", -"nodeCount" : 1, +"nodeCount" : 2, "tags" : [ { "Key": "Name", "Value": "IOOS-cloud-sandbox" }, { "Key": "Project", "Value": "IOOS-cloud-sandbox" }, { "Key": "NAME", "Value": "nosofs-fcst" } ], -"image_id" : "ami-085591407d6dd69bf", +"image_id" : "ami-01c437ac2b5beddda", "key_name" : "ioos-sandbox", "sg_ids" : [ - "sg-03e4479bc61057a30", - "sg-07588b4230c8e29b0", - "sg-03b2af1315a3fc526" + "sg-0c72b9cd2d5143515", + "sg-0fd05da3e3474d0e1", + "sg-05d4f1d9145a6666b" ], -"subnet_id" : "subnet-08689a3c6a43e807d", -"placement_group" : "Patrick-ioos-cloud-sandbox_Terraform_Placement_Group" +"subnet_id" : "subnet-0b78841b900162b6c", +"placement_group" : "Patrick-x86_64-sandbox_Terraform_Placement_Group" } diff --git a/cloudflow/cluster/configs/ioos.config b/cloudflow/cluster/configs/ioos.config index 51a4ba0..5b8f831 100644 --- a/cloudflow/cluster/configs/ioos.config +++ b/cloudflow/cluster/configs/ioos.config @@ -1,20 +1,20 @@ { "platform" : "AWS", "region" : "us-east-2", -"nodeType" : "c5n.18xlarge", -"nodeCount" : 4, +"nodeType" : "hpc6a.48xlarge", +"nodeCount" : 2, "tags" : [ { "Key": "Name", "Value": "IOOS-cloud-sandbox" }, { "Key": "Project", "Value": "IOOS-cloud-sandbox" }, { "Key": "NAME", "Value": "nosofs-fcst" } ], -"image_id" : "ami-052928467e84be78c", +"image_id" : "ami-0a4b30237fee5d223", "key_name" : "ioos-sandbox", "sg_ids" : [ - "sg-03e4479bc61057a30", - "sg-07588b4230c8e29b0", - "sg-03b2af1315a3fc526" + "sg-0810cdb0357f0a92c", + "sg-058243450eb3b4308", + "sg-0b2a044dad6a21156" ], -"subnet_id" : "subnet-08689a3c6a43e807d", -"placement_group" : "Patrick-ioos-cloud-sandbox_Terraform_Placement_Group" +"subnet_id" : "subnet-0a83f73406e809b00", +"placement_group" : "Patrick-x86_64-sandbox_Terraform_Placement_Group" } diff --git a/cloudflow/cluster/configs/ioos.slurm.cluster b/cloudflow/cluster/configs/ioos.slurm.cluster deleted file mode 100644 index 8886b61..0000000 --- a/cloudflow/cluster/configs/ioos.slurm.cluster +++ /dev/null @@ -1,20 +0,0 @@ -{ -"platform" : "AWS", -"region" : "us-east-2", -"nodeType" : "c5n.large", -"nodeCount" : 2, -"instance_ids : [ "i-0fd5f9b558b0b4024", i-0b79d8397cc7931ae" ], -"tags" : [ - { "Key": "Name", "Value": "IOOS Cloud Sandbox - SLURM Cluster Test" }, - { "Key": "Project", "Value": "IOOS-Cloud-Sandbox" } - ], -"image_id" : "ami-0c5d6c87075ac332a", -"key_name" : "ioos-sandbox", -"sg_ids" : [ - "sg-03e4479bc61057a30", - "sg-07588b4230c8e29b0", - "sg-03b2af1315a3fc526" - ], -"subnet_id" : "subnet-08689a3c6a43e807d", -"placement_group" : "" -} diff --git a/cloudflow/cluster/configs/jupyterhub_post.config b/cloudflow/cluster/configs/jupyterhub_post.config deleted file mode 100644 index b9007bb..0000000 --- a/cloudflow/cluster/configs/jupyterhub_post.config +++ /dev/null @@ -1,15 +0,0 @@ -{ -"platform" : "AWS", -"region" : "us-east-1", -"nodeType" : "t3.xlarge", -"nodeCount" : 1, -"tags" : [ - { "Key": "Name", "Value": "JupyterHub IOOS-cloud-sandbox" }, - { "Key": "Project", "Value": "IOOS-cloud-sandbox" } - ], -"image_id" : "ami-052f3e9220b9961dc", -"key_name" : "patrick-ioos-cloud-sandbox", -"sg_ids" : ["sg-006041073bfa7b072", "sg-0a48755f7b926b051", "sg-04a6bcecaec589f64", "sg-07cfe08b07d54c354"], -"subnet_id" : "subnet-050093b957519d622", -"placement_group" : "IOOS-cloud-sandbox-cluster-placement-group" -} diff --git a/cloudflow/cluster/configs/liveocean.qops.fcst b/cloudflow/cluster/configs/liveocean.qops.fcst deleted file mode 100644 index ddefd14..0000000 --- a/cloudflow/cluster/configs/liveocean.qops.fcst +++ /dev/null @@ -1,16 +0,0 @@ -{ -"platform" : "AWS", -"region" : "us-east-1", -"nodeType" : "c5n.18xlarge", -"nodeCount" : 4, -"tags" : [ - { "Key": "Name", "Value": "IOOS-cloud-sandbox" }, - { "Key": "Project", "Value": "IOOS-cloud-sandbox" }, - { "Key": "NAME", "Value": "liveocean-fcst" } - ], -"image_id" : "ami-0b1f32f8427ff563e", -"key_name" : "patrick-ioos-cloud-sandbox", -"sg_ids" : ["sg-006041073bfa7b072", "sg-0a48755f7b926b051", "sg-04a6bcecaec589f64"], -"subnet_id" : "subnet-050093b957519d622", -"placement_group" : "IOOS-cloud-sandbox-cluster-placement-group" -} diff --git a/cloudflow/cluster/configs/liveocean.qops.post b/cloudflow/cluster/configs/liveocean.qops.post deleted file mode 100644 index 9c5cccc..0000000 --- a/cloudflow/cluster/configs/liveocean.qops.post +++ /dev/null @@ -1,16 +0,0 @@ -{ -"platform" : "AWS", -"region" : "us-east-1", -"nodeType" : "t3.2xlarge", -"nodeCount" : 1, -"tags" : [ - { "Key": "Name", "Value": "IOOS-cloud-sandbox" }, - { "Key": "Project", "Value": "IOOS-cloud-sandbox" }, - { "Key": "NAME", "Value": "liveocean-post" } - ], -"image_id" : "ami-0b1f32f8427ff563e", -"key_name" : "patrick-ioos-cloud-sandbox", -"sg_ids" : ["sg-006041073bfa7b072", "sg-0a48755f7b926b051", "sg-04a6bcecaec589f64"], -"subnet_id" : "subnet-050093b957519d622", -"placement_group" : "IOOS-cloud-sandbox-cluster-placement-group" -} diff --git a/cloudflow/cluster/configs/nyh-hindcasts.config b/cloudflow/cluster/configs/nyh-hindcasts.config deleted file mode 100644 index 4838d24..0000000 --- a/cloudflow/cluster/configs/nyh-hindcasts.config +++ /dev/null @@ -1,15 +0,0 @@ -{ -"platform" : "AWS", -"region" : "us-east-1", -"nodeType" : "c5.xlarge", -"nodeCount" : 1, -"tags" : [ - { "Key": "Name", "Value": "IntecSea-Hindcasts" }, - { "Key": "Project", "Value": "IntecSea-Hindcasts" } - ], -"image_id" : "ami-01f7b31e6a631b859", -"key_name" : "forecasting", -"sg_ids" : ["sg-073fcfb0ab31a1d62","sg-0b82904b0c646072d"], -"subnet_id" : "subnet-f19221a8", -"placement_group" : "IntecSea-Placement-Group" -} diff --git a/cloudflow/cluster/configs/slurm.config b/cloudflow/cluster/configs/slurm.config deleted file mode 100644 index 55781a5..0000000 --- a/cloudflow/cluster/configs/slurm.config +++ /dev/null @@ -1,20 +0,0 @@ -{ -"platform" : "AWS", -"region" : "us-east-2", -"nodeType" : "c5n.18xlarge", -"nodeCount" : 2, -"tags" : [ - { "Key": "Name", "Value": "IOOS Cloud Sandbox - SLURM cluster test" }, - { "Key": "Project", "Value": "IOOS-cloud-sandbox" }, - { "Key": "NAME", "Value": "slurm-fcst" } - ], -"image_id" : "ami-0c5d6c87075ac332a", -"key_name" : "ioos-sandbox", -"sg_ids" : [ - "sg-03e4479bc61057a30", - "sg-07588b4230c8e29b0", - "sg-03b2af1315a3fc526" - ], -"subnet_id" : "subnet-08689a3c6a43e807d", -"placement_group" : "Patrick-ioos-cloud-sandbox_Terraform_Placement_Group" -} diff --git a/cloudflow/cluster/configs/wrfroms.config b/cloudflow/cluster/configs/wrfroms.config deleted file mode 100644 index 3b8eac4..0000000 --- a/cloudflow/cluster/configs/wrfroms.config +++ /dev/null @@ -1,16 +0,0 @@ -{ -"platform" : "AWS", -"region" : "us-east-1", -"nodeType" : "c5n.18xlarge", -"nodeCount" : 2, -"tags" : [ - { "Key": "Name", "Value": "IOOS-cloud-sandbox wrf/roms fcst" }, - { "Key": "Project", "Value": "IOOS-cloud-sandbox" }, - { "Key": "NAME", "Value": "wrfroms-fcst" } - ], -"image_id" : "ami-04c3016cfc8af41e2", -"key_name" : "patrick-ioos-cloud-sandbox", -"sg_ids" : ["sg-006041073bfa7b072", "sg-0a48755f7b926b051", "sg-04a6bcecaec589f64"], -"subnet_id" : "subnet-050093b957519d622", -"placement_group" : "IOOS-cloud-sandbox-cluster-placement-group" -} diff --git a/cloudflow/job/ROMSForecast.py b/cloudflow/job/ROMSForecast.py index 669b073..fe71c1b 100644 --- a/cloudflow/job/ROMSForecast.py +++ b/cloudflow/job/ROMSForecast.py @@ -158,7 +158,7 @@ def parseConfig(self, cfDict): if self.OCNINTMPL == "auto": self.OCNINTMPL = f"{self.TEMPLPATH}/{self.OFS}.ocean.in" - + return diff --git a/cloudflow/job/jobs/cbofs.00z.fcst b/cloudflow/job/jobs/cbofs.00z.fcst index 0cca7d0..ab63907 100644 --- a/cloudflow/job/jobs/cbofs.00z.fcst +++ b/cloudflow/job/jobs/cbofs.00z.fcst @@ -3,12 +3,12 @@ "OFS" : "cbofs", "CDATE" : "today", "HH" : "00", - "COMROT" : "/com/patrick.tripp/nosofs", - "PTMP" : "/ptmp/patrick.tripp", + "COMROT" : "/com/ec2-user/nosofs", + "PTMP" : "/ptmp/ec2-user", "EXEC" : "", "TIME_REF" : "20160101.0d0", "BUCKET" : "ioos-cloud-sandbox", - "BCKTFLDR" : "nosofs/patrick.tripp/cbofs/output", + "BCKTFLDR" : "nosofs/ec2-user/cbofs/output", "NTIMES" : "34560", "ININAME" : "", "OUTDIR" : "auto", diff --git a/cloudflow/job/jobs/cbofs.20231207 b/cloudflow/job/jobs/cbofs.20231207 new file mode 100644 index 0000000..609c7c6 --- /dev/null +++ b/cloudflow/job/jobs/cbofs.20231207 @@ -0,0 +1,17 @@ +{ + "JOBTYPE" : "romsforecast", + "OFS" : "cbofs", + "CDATE" : "20231207", + "HH" : "06", + "COMROT" : "/com/ec2-user/nosofs", + "PTMP" : "/ptmp/ec2-user", + "EXEC" : "", + "TIME_REF" : "20160101.0d0", + "BUCKET" : "ioos-cloud-sandbox", + "BCKTFLDR" : "nosofs/patrick.tripp/cbofs/output", + "NTIMES" : "34560", + "ININAME" : "", + "OUTDIR" : "auto", + "OCEANIN" : "auto", + "OCNINTMPL" : "auto" +} diff --git a/cloudflow/job/jobs/cbofs.fcst b/cloudflow/job/jobs/cbofs.fcst index 828c5b2..54ae5ab 100644 --- a/cloudflow/job/jobs/cbofs.fcst +++ b/cloudflow/job/jobs/cbofs.fcst @@ -2,9 +2,9 @@ "JOBTYPE" : "romsforecast", "OFS" : "cbofs", "CDATE" : "today", - "HH" : "06", - "COMROT" : "/com/patrick.tripp/nosofs", - "PTMP" : "/ptmp/patrick.tripp", + "HH" : "00", + "COMROT" : "/com/ec2-user/nosofs", + "PTMP" : "/ptmp/ec2-user", "EXEC" : "", "TIME_REF" : "20160101.0d0", "BUCKET" : "ioos-cloud-sandbox", diff --git a/cloudflow/job/jobs/liveocean.diffplots b/cloudflow/job/jobs/liveocean.diffplots new file mode 100644 index 0000000..a233db7 --- /dev/null +++ b/cloudflow/job/jobs/liveocean.diffplots @@ -0,0 +1,13 @@ +{ + "JOBTYPE" : "plotting_diff", + "OFS" : "liveocean", + "CDATE" : "20170601", + "HH" : "00", + "INDIR" : "/com/ec2-user/LO_roms/cas6_traps2_x2b/f2017.06.01", + "OUTDIR" : "/save/ec2-user/LO_plots", + "VERIFDIR" : "/com/ec2-user/apogee/f2017.06.01", + "VARS" : ["temp", "zeta", "w", "salt"], + "BUCKET" : "ioos-cloud-sandbox", + "BCKTFLDR" : "LiveOcean/plots", + "FSPEC" : "ocean_his_*.nc" +} diff --git a/cloudflow/job/jobs/liveocean.fcst b/cloudflow/job/jobs/liveocean.fcst index 2efaebe..a37c6ba 100644 --- a/cloudflow/job/jobs/liveocean.fcst +++ b/cloudflow/job/jobs/liveocean.fcst @@ -3,7 +3,7 @@ "OFS" : "liveocean", "CDATE" : "20170601", "HH" : "00", - "COMROT" : "/com/patrick.tripp", + "COMROT" : "/com/ec2-user", "PTMP" : "/ptmp", "EXEC" : "", "TIME_REF" : "19700101", diff --git a/cloudflow/job/templates/liveocean.ocean.in b/cloudflow/job/templates/liveocean.ocean.in index 0a1d069..0788b18 100644 --- a/cloudflow/job/templates/liveocean.ocean.in +++ b/cloudflow/job/templates/liveocean.ocean.in @@ -73,7 +73,8 @@ ! Input variable information file name. This file needs to be processed ! first so all information arrays can be initialized properly. - VARNAME = /save/ioos/patrick.tripp/LiveOcean/LO_roms_source_alt/varinfo/varinfo.yaml + #VARNAME = __SAVE__/LiveOcean/LO_roms_source_alt/varinfo/varinfo.yaml + VARNAME = varinfo.yaml ! Number of nested grids. @@ -1012,7 +1013,7 @@ PIO_I2C_Preq = 65 ! Maximum pending I2C requests NBCFILES == 1 ! number of boundary files - BRYNAME == /com/patrick.tripp/LO_output/forcing/cas6/__FDATE__/ocn00/ocean_bry.nc + BRYNAME == __COMROT__/LO_output/forcing/cas6/__FDATE__/ocn00/ocean_bry.nc ! Input climatology file names. The USER has the option to separate the ! climatology variables into individual NetCDF files (NCLMFILES > 1), diff --git a/cloudflow/plotting/plot_roms.py b/cloudflow/plotting/plot_roms.py index 9d8e7a4..e93046c 100644 --- a/cloudflow/plotting/plot_roms.py +++ b/cloudflow/plotting/plot_roms.py @@ -266,25 +266,26 @@ def plot_diff(ncfile1: str, ncfile2: str, target: str, varname: str, if __name__ == '__main__': - # source = 'figs/temp/his_arg_temp_%04d.png' - # target = 'figs/test_temp.mp4' - # png_ffmpeg(source, target) + var = 'temp' - target = '/com/nos/plots/tbofs.20200310' + # alkalinity + # salt + # oxygen + + target = '/save/ec2-user' - ncfile = '/com/nos/tbofs.20200310/nos.tbofs.fields.f048.20200310.t00z.nc' - ncfile_noaa = '/com/nos-noaa/tbofs.20200310/nos.tbofs.fields.f048.20200310.t00z.nc' + FHR="07" + myfcst='/com/ec2-user/LO_roms/cas6_traps2_x2b/f2017.06.01' + ncfile = f'{myfcst}/ocean_his_00{FHR}.nc' - #ncfile = '/com/nos/tbofs.20200310/nos.tbofs.fields.f001.20200310.t00z.nc' - #ncfile_noaa = '/com/nos-noaa/tbofs.20200310/nos.tbofs.fields.f001.20200310.t00z.nc' + otherfcst='/com/ec2-user/apogee/f2017.06.01' + ncfile_noaa = f'{otherfcst}/ocean_his_00{FHR}.nc' if not os.path.exists(target): os.makedirs(target) - # ncfile='/com/liveocean/f2020.02.13/ocean_his_0001.nc' - # target='/com/liveocean/plots/f2020.02.13' # plot_roms(ncfile, target, var, True, 8) - #plot(ncfile, target, var) + # plot(ncfile, target, var) plot_diff(ncfile_noaa, ncfile, target, var, vmin=-0.5, vmax=0.5) diff --git a/cloudflow/requirements.txt b/cloudflow/requirements.txt index fb8db84..bd5e8b6 100644 --- a/cloudflow/requirements.txt +++ b/cloudflow/requirements.txt @@ -8,3 +8,5 @@ dask distributed Pillow boto3 +cartopy +xarray diff --git a/cloudflow/setup.py b/cloudflow/setup.py index 879de34..6d49086 100644 --- a/cloudflow/setup.py +++ b/cloudflow/setup.py @@ -19,5 +19,8 @@ 'netCDF4', 'dask', 'distributed', - 'Pillow'] - ) + 'Pillow', + 'boto3', + 'cartopy', + 'xarray'] +) diff --git a/cloudflow/workflows/fcst_launcher.sh b/cloudflow/workflows/fcst_launcher.sh index a87e501..a8e1575 100755 --- a/cloudflow/workflows/fcst_launcher.sh +++ b/cloudflow/workflows/fcst_launcher.sh @@ -2,7 +2,7 @@ # Script used to launch forecasts. # BASH is used in order to bridge between the Python interface and NCO's BASH based run scripts -WRKDIR=/save/$GROUP/$USER +WRKDIR=/save/$USER set -xa ulimit -c unlimited diff --git a/cloudflow/workflows/flows.py b/cloudflow/workflows/flows.py index 62c5639..564da7e 100755 --- a/cloudflow/workflows/flows.py +++ b/cloudflow/workflows/flows.py @@ -152,7 +152,7 @@ def fcst_flow(fcstconf, fcstjobfile, sshuser) -> Flow: # Terminate the cluster nodes cluster_stop = ctasks.cluster_terminate(cluster, upstream_tasks=[fcst_run]) - # Copy the results to /com (liveocean) + # Copy the results to /com (liveocean does not run in ptmp currently) cp2com = jtasks.ptmp2com(fcstjob, upstream_tasks=[fcst_run]) # Copy the results to S3 (optionally) @@ -162,13 +162,13 @@ def fcst_flow(fcstconf, fcstjobfile, sshuser) -> Flow: #pngtocloud.set_upstream(plots) # Copy the results to S3 (optionally. currently only saves LiveOcean) - # storage_service = tasks.storage_init(provider) - # cp2cloud = tasks.save_history(fcstjob, storage_service, ['*.nc'], public=True, upstream_tasks=[storage_service,cp2com]) + #storage_service = tasks.storage_init(provider) + #cp2cloud = tasks.save_history(fcstjob, storage_service, ['*.nc'], public=True, upstream_tasks=[storage_service,cp2com]) #pngtocloud.set_upstream(plots) # If the fcst fails, then set the whole flow to fail - fcstflow.set_reference_tasks([fcst_run, cp2com]) + fcstflow.set_reference_tasks([fcst_run]) return fcstflow @@ -277,10 +277,11 @@ def diff_plot_flow(postconf, postjobfile, sshuser=None) -> Flow: # Push the env, install required libs on post machine # TODO: install all of the 3rd party dependencies on AMI - pushPy = ctasks.push_pyEnv(postmach, upstream_tasks=[pmStarted]) + #pushPy = ctasks.push_pyEnv(postmach, upstream_tasks=[pmStarted]) # Start a dask scheduler on the new post machine - daskclient: Client = ctasks.start_dask(postmach, upstream_tasks=[pushPy]) + #daskclient: Client = ctasks.start_dask(postmach, upstream_tasks=[pushPy]) + daskclient: Client = ctasks.start_dask(postmach) # Get list of files from job specified directory FILES = jtasks.ncfiles_from_Job(plotjob) @@ -290,24 +291,27 @@ def diff_plot_flow(postconf, postjobfile, sshuser=None) -> Flow: plots = jtasks.daskmake_diff_plots(daskclient, FILES, BASELINE, plotjob) plots.set_upstream([daskclient, getbaseline]) - storage_service = tasks.storage_init(provider) + # storage_service = tasks.storage_init(provider) #pngtocloud = tasks.save_to_cloud(plotjob, storage_service, ['*diff.png'], public=True) #pngtocloud.set_upstream(plots) # Make movies - mpegs = jtasks.daskmake_mpegs(daskclient, plotjob, diff=True, upstream_tasks=[plots]) - mp4tocloud = tasks.save_to_cloud(plotjob, storage_service, ['*diff.mp4'], public=True) - mp4tocloud.set_upstream(mpegs) + #mpegs = jtasks.daskmake_mpegs(daskclient, plotjob, diff=True, upstream_tasks=[plots]) + #mp4tocloud = tasks.save_to_cloud(plotjob, storage_service, ['*diff.mp4'], public=True) + #mp4tocloud.set_upstream(mpegs) - closedask = ctasks.dask_client_close(daskclient, upstream_tasks=[mpegs]) - pmTerminated = ctasks.cluster_terminate(postmach, upstream_tasks=[mpegs, closedask]) + # closedask = ctasks.dask_client_close(daskclient, upstream_tasks=[mpegs]) + # pmTerminated = ctasks.cluster_terminate(postmach, upstream_tasks=[mpegs, closedask]) + + closedask = ctasks.dask_client_close(daskclient, upstream_tasks=[plots]) + pmTerminated = ctasks.cluster_terminate(postmach, upstream_tasks=[plots, closedask]) ####################################################################### # This will add Kenny's script # https://ioos-cloud-sandbox.s3.amazonaws.com/cloudflow/inject/kenny/cloud_sandbot.py #notebook = 'cloudflow/inject/kenny/cloud_sandbot.py' - notebook = 'cloudflow/inject/patrick/sandbot_current_fcst.py' - injected = tasks.fetchpy_and_run(plotjob, storage_service, notebook) + #notebook = 'cloudflow/inject/patrick/sandbot_current_fcst.py' + #injected = tasks.fetchpy_and_run(plotjob, storage_service, notebook) return diff_plotflow diff --git a/cloudflow/workflows/scripts/getICsFVCOM.sh b/cloudflow/workflows/scripts/getICsFVCOM.sh index 6548a1e..665c039 100755 --- a/cloudflow/workflows/scripts/getICsFVCOM.sh +++ b/cloudflow/workflows/scripts/getICsFVCOM.sh @@ -5,6 +5,7 @@ . /usr/share/Modules/init/sh +module use -a /usrx/modulefiles module load produtil module load gcc diff --git a/cloudflow/workflows/scripts/getICsROMS.sh b/cloudflow/workflows/scripts/getICsROMS.sh index 54f2fb2..0d9806d 100755 --- a/cloudflow/workflows/scripts/getICsROMS.sh +++ b/cloudflow/workflows/scripts/getICsROMS.sh @@ -6,8 +6,8 @@ set -x . /usr/share/Modules/init/sh +module use -a /usrx/modulefiles module load produtil -# module load gcc if [ $# -ne 4 ] ; then echo "Usage: $0 YYYYMMDD HH cbofs|(other ROMS model) COMDIR" diff --git a/cloudflow/workflows/scripts/get_fixfiles_s3.sh b/cloudflow/workflows/scripts/get_fixfiles_s3.sh old mode 100755 new mode 100644 diff --git a/cloudflow/workflows/scripts/scp-secret.sh b/cloudflow/workflows/scripts/scp-secret.sh deleted file mode 100755 index a069920..0000000 --- a/cloudflow/workflows/scripts/scp-secret.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -host='ec2-3-132-69-122.us-east-2.compute.amazonaws.com' -user='centos' - -scp -p -i ioos-sandbox.pem ~/RPS/SPACK/spack.mirror.gpgkey.secret ${user}@${host}:/save/environments/spack/opt/spack/gpg - -ssh -i ioos-sandbox.pem ${user}@${host} 'cd /save/environments/spack/opt/spack/gpg; spack gpg trust spack.mirror.gpgkey.secret' diff --git a/cloudflow/workflows/scripts/setup-instance.sh b/cloudflow/workflows/scripts/setup-instance.sh deleted file mode 100755 index 8da1c90..0000000 --- a/cloudflow/workflows/scripts/setup-instance.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env bash - -#__copyright__ = "Copyright © 2023 RPS Group, Inc. All rights reserved." -#__license__ = "BSD 3-Clause" - - -GCC_VER=8.5.0 -INTEL_VER=2021.3.0 - -SPACK_VER='releases/v0.18' -#SPACK_DIR='/mnt/efs/fs1/opt/spack' -SPACK_DIR='/save/environments/spack' -SPACKOPTS='-v -y' -SPACKTARGET='target=skylake_avx512' -#export SPACKTARGET='target=haswell' # for AMD nodes - -SLURM_VER='22-05-2-1' - -# 1 = Don't build any packages. Only install packages from binary mirrors -# 0 = Will build if not found in mirror/cache -# -1 = Don't check pre-built binary cache -SPACK_CACHEONLY=0 - -# source include the functions -. funcs-setup-instance.sh - -# calling sudo from cloud init adds 25 second delay for each sudo -sudo setenforce 0 - -# Use caution when changing the order of the following - -# System stuff -setup_environment -setup_paths -setup_aliases -# install_jupyterhub # Not fully tested yet -setup_ssh_mpi -install_efa_driver - -# Compilers and libraries -install_python_modules_user -install_spack -install_gcc -install_intel_oneapi -install_esmf -install_base_rpms -install_extra_rpms -install_ffmpeg - - -# Compute node config -install_slurm - -configure_slurm compute -sudo yum -y clean all - -# TODO: create an output file to contain all of this state info - json -# TODO: re-write in Python ? - -## Create the AMI to be used for the compute nodes -# TODO: make the next section cleaner, more abstracted away - -export AWS_DEFAULT_REGION=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone | sed 's/[a-z]$//') -instance_id=`curl -s http://169.254.169.254/latest/meta-data/instance-id` - -# ami_name is provided by Terraform if called via the init_template -# otherwise it will use the default - -ami_name=${ami_name:='IOOS-Cloud-Sandbox'} - -# TODO: pass this in via Terraform init template -project_tag="IOOS-Cloud-Sandbox" - -image_name="${ami_name}-Compute-Node" -echo "Compute node image_name: '$image_name'" - -# For some reason these aren't being seen -# Try installing them again in this shell -python3 -m pip install --user --upgrade botocore==1.23.46 -python3 -m pip install --user --upgrade boto3==1.20.46 - -# Flush the disk cache -sudo sync - -image_id=`python3 create_image.py $instance_id "$image_name" "$project_tag"` -echo "Compute node image_id: $image_id" - -# Configure this machine as a head node -configure_slurm head -sudo yum -y clean all - -# Optionally create Head node image -################################### - -image_name="${ami_name}-Head-Node" -echo "Head node image_name: $image_name" - -sudo sync -image_id=`python3 create_image.py $instance_id "$image_name" "$project_tag"` -echo "Head node image_id: $image_id" - -echo "Setup completed!" diff --git a/cloudflow/workflows/scripts/test-setup.sh b/cloudflow/workflows/scripts/test-setup.sh deleted file mode 100755 index 20fcf82..0000000 --- a/cloudflow/workflows/scripts/test-setup.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env bash - -#__copyright__ = "Copyright © 2023 RPS Group, Inc. All rights reserved." -#__license__ = "BSD 3-Clause" - - -GCC_VER=8.5.0 -INTEL_VER=2021.3.0 - -SPACK_VER='releases/v0.18' -#SPACK_DIR='/mnt/efs/fs1/opt/spack' -SPACK_DIR='/save/environments/spack' -SPACKOPTS='-v -y' - -SLURM_VER='22-05-2-1' - -# 1 = Don't build any packages. Only install packages from binary mirrors -# 0 = Will build if not found in mirror/cache -# -1 = Don't check pre-built binary cache -SPACK_CACHEONLY=0 - -# source include the functions -. funcs-setup-instance.sh - -# calling sudo from cloud init adds 25 second delay for each sudo -sudo setenforce 0 - -# Use caution when changing the order of the following - -# System stuff -# setup_environment -# setup_paths -# setup_aliases -# install_jupyterhub # Not fully tested yet -# setup_ssh_mpi -# install_efa_driver - -# Compilers and libraries -# install_python_modules_user -# install_spack -# install_gcc -# install_intel_oneapi -install_esmf -# install_base_rpms -# install_extra_rpms -# install_ffmpeg - -exit - -# Compute node config -# install_slurm - -# configure_slurm compute -# sudo yum -y clean all - -# TODO: create an output file to contain all of this state info - json -# TODO: re-write in Python ? - -## Create the AMI to be used for the compute nodes -# TODO: make the next section cleaner, more abstracted away - -export AWS_DEFAULT_REGION=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone | sed 's/[a-z]$//') -instance_id=`curl -s http://169.254.169.254/latest/meta-data/instance-id` - -# ami_name is provided by Terraform if called via the init_template -# otherwise it will use the default - -ami_name=${ami_name:='IOOS-Cloud-Sandbox'} - -# TODO: pass this in via Terraform init template -project_tag="IOOS-Cloud-Sandbox" - -image_name="${ami_name}-Compute-Node" -echo "Compute node image_name: '$image_name'" - -# For some reason these aren't being seen -# Try installing them again in this shell -python3 -m pip install --user --upgrade botocore==1.23.46 -python3 -m pip install --user --upgrade boto3==1.20.46 - -# Flush the disk cache -sudo sync - -image_id=`python3 create_image.py $instance_id "$image_name" "$project_tag"` -echo "Compute node image_id: $image_id" - -# Configure this machine as a head node -configure_slurm head -sudo yum -y clean all - -# Optionally create Head node image -################################### - -image_name="${ami_name}-Head-Node" -echo "Head node image_name: $image_name" - -sudo sync -image_id=`python3 create_image.py $instance_id "$image_name" "$project_tag"` -echo "Head node image_id: $image_id" - -echo "Setup completed!" diff --git a/cloudflow/workflows/workflow_main.py b/cloudflow/workflows/workflow_main.py index 8dcb2ca..7d5aa6d 100755 --- a/cloudflow/workflows/workflow_main.py +++ b/cloudflow/workflows/workflow_main.py @@ -28,7 +28,9 @@ curdir = os.path.dirname(os.path.abspath(__file__)) fcstconf = f'{curdir}/../cluster/configs/ioos.config' -postconf = f'{curdir}/../cluster/configs/post.config' +#postconf = f'{curdir}/../cluster/configs/post.config' +postconf = f'{curdir}/../cluster/configs/local.config' +#postconf = f'{curdir}/../cluster/configs/ioos.config' # This is used for obtaining liveocean forcing data # LiveOcean users need to obtain credentials from UW diff --git a/modulefiles/compiler-rt/2023.1.0 b/modulefiles/compiler-rt/2023.1.0 new file mode 100644 index 0000000..b27d3e5 --- /dev/null +++ b/modulefiles/compiler-rt/2023.1.0 @@ -0,0 +1 @@ +/opt/intel/oneapi/compiler/2023.1.0/modulefiles/compiler-rt \ No newline at end of file diff --git a/modulefiles/compiler-rt32/2023.1.0 b/modulefiles/compiler-rt32/2023.1.0 new file mode 100644 index 0000000..59fe949 --- /dev/null +++ b/modulefiles/compiler-rt32/2023.1.0 @@ -0,0 +1 @@ +/opt/intel/oneapi/compiler/2023.1.0/modulefiles/compiler-rt32 \ No newline at end of file diff --git a/modulefiles/compiler/2023.1.0 b/modulefiles/compiler/2023.1.0 new file mode 100644 index 0000000..60632f9 --- /dev/null +++ b/modulefiles/compiler/2023.1.0 @@ -0,0 +1 @@ +/opt/intel/oneapi/compiler/2023.1.0/modulefiles/compiler \ No newline at end of file diff --git a/modulefiles/compiler32/2023.1.0 b/modulefiles/compiler32/2023.1.0 new file mode 100644 index 0000000..fb6272c --- /dev/null +++ b/modulefiles/compiler32/2023.1.0 @@ -0,0 +1 @@ +/opt/intel/oneapi/compiler/2023.1.0/modulefiles/compiler32 \ No newline at end of file diff --git a/modulefiles/debugger/2023.1.0 b/modulefiles/debugger/2023.1.0 new file mode 100644 index 0000000..c62fac2 --- /dev/null +++ b/modulefiles/debugger/2023.1.0 @@ -0,0 +1 @@ +/opt/intel/oneapi/debugger/2023.1.0/modulefiles/debugger \ No newline at end of file diff --git a/modulefiles/dev-utilities/2021.9.0 b/modulefiles/dev-utilities/2021.9.0 new file mode 100644 index 0000000..2d7c845 --- /dev/null +++ b/modulefiles/dev-utilities/2021.9.0 @@ -0,0 +1 @@ +/opt/intel/oneapi/dev-utilities/2021.9.0/modulefiles/dev-utilities \ No newline at end of file diff --git a/modulefiles/icc/2023.1.0 b/modulefiles/icc/2023.1.0 new file mode 100644 index 0000000..0e38c88 --- /dev/null +++ b/modulefiles/icc/2023.1.0 @@ -0,0 +1 @@ +/opt/intel/oneapi/compiler/2023.1.0/modulefiles/icc \ No newline at end of file diff --git a/modulefiles/icc32/2023.1.0 b/modulefiles/icc32/2023.1.0 new file mode 100644 index 0000000..613fa26 --- /dev/null +++ b/modulefiles/icc32/2023.1.0 @@ -0,0 +1 @@ +/opt/intel/oneapi/compiler/2023.1.0/modulefiles/icc32 \ No newline at end of file diff --git a/modulefiles/init_opencl/2023.1.0 b/modulefiles/init_opencl/2023.1.0 new file mode 100644 index 0000000..fe806b1 --- /dev/null +++ b/modulefiles/init_opencl/2023.1.0 @@ -0,0 +1 @@ +/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga/modulefiles/init_opencl \ No newline at end of file diff --git a/modulefiles/mpi/2021.9.0 b/modulefiles/mpi/2021.9.0 new file mode 100644 index 0000000..79e65a6 --- /dev/null +++ b/modulefiles/mpi/2021.9.0 @@ -0,0 +1 @@ +/opt/intel/oneapi/mpi/2021.9.0/modulefiles/mpi \ No newline at end of file diff --git a/modulefiles/oclfpga/2023.1.0 b/modulefiles/oclfpga/2023.1.0 new file mode 100644 index 0000000..3111196 --- /dev/null +++ b/modulefiles/oclfpga/2023.1.0 @@ -0,0 +1 @@ +/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga/modulefiles/oclfpga \ No newline at end of file diff --git a/modulefiles/tbb/2021.9.0 b/modulefiles/tbb/2021.9.0 new file mode 100644 index 0000000..95cb060 --- /dev/null +++ b/modulefiles/tbb/2021.9.0 @@ -0,0 +1 @@ +/opt/intel/oneapi/tbb/2021.9.0/modulefiles/tbb \ No newline at end of file diff --git a/cloudflow/workflows/scripts/add_group.sh b/scripts/add_group.sh similarity index 100% rename from cloudflow/workflows/scripts/add_group.sh rename to scripts/add_group.sh diff --git a/cloudflow/workflows/scripts/add_user.sh b/scripts/add_user.sh similarity index 100% rename from cloudflow/workflows/scripts/add_user.sh rename to scripts/add_user.sh diff --git a/cloudflow/workflows/scripts/aws_ec2meta.sh b/scripts/aws_ec2meta.sh similarity index 100% rename from cloudflow/workflows/scripts/aws_ec2meta.sh rename to scripts/aws_ec2meta.sh diff --git a/cloudflow/workflows/scripts/batch_spack_uninstall.sh b/scripts/batch_spack_uninstall.sh similarity index 90% rename from cloudflow/workflows/scripts/batch_spack_uninstall.sh rename to scripts/batch_spack_uninstall.sh index 353ceb7..e46d42f 100755 --- a/cloudflow/workflows/scripts/batch_spack_uninstall.sh +++ b/scripts/batch_spack_uninstall.sh @@ -6,12 +6,13 @@ # intel@2021.3.0 #ARCH='linux-centos7-core2' -ARCH='linux-centos7-skylake_avx512' +# ARCH='linux-centos7-skylake_avx512' #COMP='intel@2021.3.0' #COMP='gcc@8.5.0' -COMP='gcc@4.8.5' +# COMP='gcc@4.8.5' +COMP='gcc@11.4.1' -TARGET='skylake_avx512' +# TARGET='skylake_avx512' # TARGET='haswell' # TARGET='broadwell' diff --git a/cloudflow/workflows/scripts/build_gcc.sh b/scripts/build_gcc.sh similarity index 100% rename from cloudflow/workflows/scripts/build_gcc.sh rename to scripts/build_gcc.sh diff --git a/cloudflow/workflows/scripts/buildit.sh b/scripts/buildit.sh similarity index 100% rename from cloudflow/workflows/scripts/buildit.sh rename to scripts/buildit.sh diff --git a/cloudflow/workflows/scripts/change-instance-type.sh b/scripts/change-instance-type.sh similarity index 100% rename from cloudflow/workflows/scripts/change-instance-type.sh rename to scripts/change-instance-type.sh diff --git a/cloudflow/workflows/scripts/create_image.py b/scripts/create_image.py similarity index 78% rename from cloudflow/workflows/scripts/create_image.py rename to scripts/create_image.py index 836ff3d..9346e6f 100755 --- a/cloudflow/workflows/scripts/create_image.py +++ b/scripts/create_image.py @@ -6,7 +6,7 @@ import boto3 from botocore.exceptions import ClientError -DEBUG=False +DEBUG=True def main(): @@ -22,9 +22,20 @@ def main(): print(f"Usage: {sys.argv[0]} ") sys.exit(1) + #print (f"DEBUG: instance_id: {instance_id}, image_name: {image_name}, project_tag: {project_tag}") + + + print("Creating a snapshot of current root volume ...") snapshot_id = create_snapshot(instance_id, image_name, project_tag) - image_id = create_image_from_snapshot(snapshot_id, image_name) - print(str(image_id)) + if snapshot_id == None: + print("ERROR: could not create snapshot") + sys.exit(1) + + print(f"snapshot_id: {snapshot_id}") + + print("Creating AMI from snapshot ...") + image_id = create_image_from_snapshot(snapshot_id, image_name) + print("image_id: ", str(image_id)) def create_snapshot(instance_id: str, name_tag: str, project_tag: str): @@ -32,7 +43,7 @@ def create_snapshot(instance_id: str, name_tag: str, project_tag: str): # print(f"create_snapshot: instance_id: {instance_id}, name_tag: {name_tag}, project_tag: {project_tag}") - ec2 = boto3.client('ec2') + ec2 = boto3.client('ec2', region_name='us-east-2') description = f"Created by IOOS Cloud Sandbox for AMI" try: @@ -51,12 +62,12 @@ def create_snapshot(instance_id: str, name_tag: str, project_tag: str): ] } ], - DryRun=False, - CopyTagsFromSource='volume' + DryRun=False + # CopyTagsFromSource='volume' ) except Exception as e: - # print(str(e)) - if DEBUG: traceback.print_stack() + print(str(e)) + traceback.print_stack() return None snapshot_id = response['Snapshots'][0]['SnapshotId'] @@ -90,18 +101,19 @@ def create_image_from_snapshot(snapshot_id: str, image_name: str): # Wait for snapshot to be created # wait_until will throw an exception after 10 minutes - maxtries=2 - tries=0 - while tries < maxtries: + maxtries=5 + tries=1 + while tries <= maxtries: try: snapshot.wait_until_completed() break except Exception as e: - #print("Exception: " + str(e)) + print("WARNING: " + str(e)) tries += 1 + print(f"tries {tries} of {maxtries}") if tries == maxtries: - # print("ERROR: maxtries reached. something went wrong") - if DEBUG: traceback.print_stack() + print("ERROR: maxtries reached. something went wrong") + traceback.print_stack() return None diff --git a/scripts/create_image.sh b/scripts/create_image.sh new file mode 100755 index 0000000..86b41cf --- /dev/null +++ b/scripts/create_image.sh @@ -0,0 +1,34 @@ +#!/bin/env bash +#set -x + +ami_name=$1 +project_tag=$2 + +#echo "Requesting TOKEN..." +TOKEN=`curl -sX PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"` + +#echo "Getting current region..." +export AWS_DEFAULT_REGION=`curl -sH "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/placement/availability-zone | sed 's/[a-z]$//'` +#echo "AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION" + +#echo "Getting current instance id..." +instance_id=`curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-id` +# echo "instance_id: $instance_id" + +ami_name=${ami_name:='IOOS-Cloud-Sandbox'} +# echo "ami_name: $ami_name" + +# TODO: pass this in via Terraform init template +project_tag=${project_tag:="IOOS-Cloud-Sandbox"} +# echo "project_tag: $project_tag" + +# create node image +################################### + +image_name="${ami_name}-Node" +echo "Node image_name: $image_name" + +# Flush the disk cache +sudo sync +python3 create_image.py $instance_id "$image_name" "$project_tag" + diff --git a/cloudflow/workflows/scripts/disableHT.sh b/scripts/disableHT.sh similarity index 100% rename from cloudflow/workflows/scripts/disableHT.sh rename to scripts/disableHT.sh diff --git a/cloudflow/workflows/scripts/funcs-setup-instance.sh b/scripts/funcs-setup-instance.sh similarity index 72% rename from cloudflow/workflows/scripts/funcs-setup-instance.sh rename to scripts/funcs-setup-instance.sh index a1aa2d2..61d7f6f 100644 --- a/cloudflow/workflows/scripts/funcs-setup-instance.sh +++ b/scripts/funcs-setup-instance.sh @@ -3,9 +3,9 @@ # -1 = Don't check pre-built binary cache if [ $SPACK_CACHEONLY -eq 1 ]; then - SPACKOPTS="$SPACKOPS --cache-only" + SPACKOPTS="$SPACKOPTS --cache-only" elif [ $SPACK_CACHEONLY -eq -1 ]; then - SPACKOPTS="$SPACKOPS --no-cache" + SPACKOPTS="$SPACKOPTS --no-cache" fi # This script will setup the required system components, libraries @@ -15,9 +15,10 @@ fi #__license__ = "BSD 3-Clause" - setup_environment () { +# This has been tested on RHEL8 + echo "Running ${FUNCNAME[0]} ..." home=$PWD @@ -25,11 +26,16 @@ setup_environment () { # By default, centos 7 does not install the docs (man pages) for packages, remove that setting here sudo sed -i 's/tsflags=nodocs/# &/' /etc/yum.conf + # Get rid of subscription manager messages + sudo subscription-manager config --rhsm.manage_repos=0 + sudo sed -i 's/enabled[ ]*=[ ]*1/enabled=0/g' /etc/yum/pluginconf.d/subscription-manager.conf + + sudo yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm + # yum update might update the kernel. # This might cause some of the other installs to fail, e.g. efa driver - #sudo yum -y update - sudo yum -y install epel-release + sudo yum -y install tcsh sudo yum -y install ksh sudo yum -y install wget @@ -42,16 +48,29 @@ setup_environment () { sudo yum -y install bzip2-devel sudo yum -y install automake sudo yum -y install vim-enhanced - sudo yum -y install environment-modules - sudo yum -y install python3-devel + sudo yum -y install subversion + + sudo yum -y install python3.11-devel + sudo alternatives --set python3 /usr/bin/python3.11 + sudo yum -y install python3.11-pip sudo yum -y install jq - cliver="2.2.10" + # Additional packages for spack-stack + #sudo yum -y install git-lfs + #sudo yum -y install bash-completion + #sudo yum -y install xorg-x11-xauth + #sudo yum -y install xterm + #sudo yum -y install texlive + #sudo yum -y install mysql-server + + cliver="2.10.0" curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-${cliver}.zip" -o "awscliv2.zip" /usr/bin/unzip -q awscliv2.zip sudo ./aws/install rm awscliv2.zip - rm -Rf "./aws" + sudo rm -Rf "./aws" + + sudo yum -y install environment-modules # Only do this once grep "/usr/share/Modules/init/bash" ~/.bashrc >& /dev/null @@ -62,12 +81,22 @@ setup_environment () { fi # Only do this once - if [ ! -d /usrx/modulefiles ] ; then - sudo mkdir -p /usrx/modulefiles - echo /usrx/modulefiles | sudo tee -a ${MODULESHOME}/init/.modulespath + if [ ! -d /save/environments/modulefiles ] ; then + sudo mkdir -p /save/environments/modulefiles + echo "/save/environments/modulefiles" | sudo tee -a ${MODULESHOME}/init/.modulespath + echo "/usrx/modulefiles" | sudo tee -a ${MODULESHOME}/init/.modulespath echo ". /usr/share/Modules/init/bash" | sudo tee -a /etc/profile.d/custom.sh echo "source /usr/share/Modules/init/csh" | sudo tee -a /etc/profile.d/custom.csh + echo "module use -a /usrx/modulefiles" >> ~/.bashrc + . ~/.bashrc fi + + # Add unlimited stack size + echo "ulimit -s unlimited" | sudo tee -a /etc/profile.d/custom.sh + + # sudo yum clean {option} + cd $home + } @@ -75,6 +104,7 @@ setup_paths () { echo "Running ${FUNCNAME[0]} ..." + set -x home=$PWD if [ ! -d /mnt/efs/fs1 ]; then @@ -83,23 +113,34 @@ setup_paths () { fi cd /mnt/efs/fs1 + if [ ! -d ptmp ] ; then sudo mkdir ptmp - sudo mkdir com - sudo mkdir save - sudo chgrp wheel ptmp sudo chmod 777 ptmp + sudo ln -s /mnt/efs/fs1/ptmp /ptmp + fi + + if [ ! -d com ] ; then + sudo mkdir com sudo chgrp wheel com sudo chmod 777 com - sudo chgrp wheel save - sudo chmod 777 save + sudo ln -s /mnt/efs/fs1/com /com fi - sudo ln -s /mnt/efs/fs1/ptmp /ptmp - sudo ln -s /mnt/efs/fs1/com /com - sudo ln -s /mnt/efs/fs1/save /save +# Not sure why it keeps creating an extra sym link +# if [ ! -d save ] ; then +# sudo mkdir save +# sudo chgrp wheel save +# sudo chmod 777 save +# sudo ln -s /mnt/efs/fs1/save /save +# fi + + # mkdir /save/$USER + mkdir /com/$USER + mkdir /ptmp/$USER + set +x cd $home } @@ -121,33 +162,10 @@ install_efa_driver() { # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-start.html -# Note this error: -# No package kernel-devel-3.10.0-1062.12.1.el7.x86_64 available. -# Error: Not tolerating missing names on install, stopping. -# Error: Failed to install packages. -# -# ============================================================================== -# The kernel header of the current kernel version cannot be installed and is required -# to build the EFA kernel module. Please install the kernel header package for your -# distribution manually or build the EFA kernel driver manually and re-run the installer -# with --skip-kmod. -# ============================================================================== -# -# The kernel version might have been updated since last reboot, if so, reboot the machine, and rerun this step. -# To see the current running kernel: -# uname -a -# -# Available kernels are visible at /usr/lib/modules -# -# The AWS centos 7 is still at version 1062 and has to be updated and rebooted before this will work -# since the standard yum registry only has the current 1160 kernel-devel package -# 3.10.0-1160.24.1.el7.x86_64 - home=$PWD version=latest - #version=1.11.2 (April 20, 2021) - #version=1.8.3 + # version=1.14.1 # Last one with CentOS 8 support tarfile=aws-efa-installer-${version}.tar.gz wrkdir=~/efadriver @@ -167,8 +185,6 @@ install_efa_driver() { # Default version is needed to build the kernel driver # If gcc has already been upgraded, this will likely fail - # Should uninstall newer one and install the default 4.8 - # sudo yum -y install gcc curl -s -O https://s3-us-west-2.amazonaws.com/aws-efa-installer/$tarfile tar -xf $tarfile @@ -193,13 +209,52 @@ install_efa_driver() { #-----------------------------------------------------------------------------# +install_gcc_toolset_yum() { + + echo "Running ${FUNCNAME[0]} ..." + + home=$PWD + + sudo yum -y install gcc-toolset-11-gcc-c++ + sudo yum -y install gcc-toolset-11-gcc-gfortran + sudo yum -y install gcc-toolset-11-gdb + + # scl enable gcc-toolset-11 bash - Not inside a script + # source scl_source enable gcc-toolset-11 + # source /opt/rh/gcc-toolset-11/enable + cd $home +} + +#-----------------------------------------------------------------------------# + +install_spack-stack() { + + echo "Running ${FUNCNAME[0]} ..." + home=$PWD + + SPACK_MIRROR='s3://ioos-cloud-sandbox/public/spack/mirror' + SPACK_KEY_URL='https://ioos-cloud-sandbox.s3.amazonaws.com/public/spack/mirror/spack.mirror.gpgkey.pub' + SPACK_KEY="$SPACK_DIR/opt/spack/gpg/spack.mirror.gpgkey.pub" + + cd /save/environments + git clone --recurse-submodules -b ioos-aws https://github.com/asascience/spack-stack.git + cd spack-stack + source setup.sh + + # To be continued .... + +} + +#-----------------------------------------------------------------------------# + install_spack() { echo "Running ${FUNCNAME[0]} ..." home=$PWD + source /opt/rh/gcc-toolset-11/enable + SPACK_MIRROR='s3://ioos-cloud-sandbox/public/spack/mirror' - #SPACK_MIRROR='https://ioos-cloud-sandbox.s3.amazonaws.com/public/spack/mirror' SPACK_KEY_URL='https://ioos-cloud-sandbox.s3.amazonaws.com/public/spack/mirror/spack.mirror.gpgkey.pub' SPACK_KEY="$SPACK_DIR/opt/spack/gpg/spack.mirror.gpgkey.pub" @@ -210,7 +265,8 @@ install_spack() { return fi - mkdir -p $SPACK_DIR + sudo mkdir -p $SPACK_DIR + sudo chown ec2-user:ec2-user $SPACK_DIR git clone -q https://github.com/spack/spack.git $SPACK_DIR cd $SPACK_DIR git checkout -q $SPACK_VER @@ -219,25 +275,33 @@ install_spack() { # Location for overriding default configurations sudo mkdir /etc/spack + sudo chown ec2-user:ec2-user /etc/spack . $SPACK_DIR/share/spack/setup-env.sh # TODO: Rebuild everything using this, and push to mirror # spack config add "config:install_tree:padded_length:128" + spack config add "modules:default:enable:[tcl]" # Using an s3-mirror for previously built packages echo "Using SPACK s3-mirror $SPACK_MIRROR" spack mirror add s3-mirror $SPACK_MIRROR spack buildcache keys --install --trust --force - spack buildcache update-index -d $SPACK_MIRROR + spack buildcache update-index $SPACK_MIRROR + + spack compiler find --scope system + + # Note to recreate modulefiles + # spack module tcl refresh -y cd $home } + #-----------------------------------------------------------------------------# -install_gcc () { +install_gcc_spack () { echo "Running ${FUNCNAME[0]} ..." @@ -248,7 +312,7 @@ install_gcc () { . $SPACK_DIR/share/spack/setup-env.sh # TODO: Rebuild all using x86_64 - spack install $SPACKOPTS gcc@$GCC_VER + spack install $SPACKOPTS --no-checksum gcc@$GCC_VER ^ncurses@6.4 $SPACKTARGET spack compiler add `spack location -i gcc@$GCC_VER`/bin @@ -262,20 +326,107 @@ install_gcc () { #-----------------------------------------------------------------------------# -install_intel_oneapi () { +install_intel_oneapi-spack-stack () { echo "Running ${FUNCNAME[0]} ..." home=$PWD - . $SPACK_DIR/share/spack/setup-env.sh + cd $SPACK_DIR + source setup.sh + SPACK_ENV=ioos-aws-rhel + + cd $SPACK_ENV + spack env activate -p . spack install $SPACKOPTS intel-oneapi-compilers@${INTEL_VER} + spack compiler add `spack location -i intel-oneapi-compilers`/compiler/${INTEL_VER}/linux/bin/intel64 + spack compiler add `spack location -i intel-oneapi-compilers`/compiler/${INTEL_VER}/linux/bin + + spack install $SPACKOPTS intel-oneapi-mpi@${INTEL_VER} %intel@${INTEL_VER} + spack install $SPACKOPTS intel-oneapi-mkl@${INTEL_VER} %intel@${INTEL_VER} + + cd $home +} + +#-----------------------------------------------------------------------------# + +install_intel_oneapi_yum () { + + echo "Running ${FUNCNAME[0]} ..." + + home=$PWD + +tee > /tmp/oneAPI.repo << EOF +[oneAPI] +name=Intel® oneAPI repository +baseurl=https://yum.repos.intel.com/oneapi +enabled=1 +gpgcheck=0 +repo_gpgcheck=0 +gpgkey=https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB +EOF + +sudo mv /tmp/oneAPI.repo /etc/yum.repos.d + +sudo rpm --import https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB + +# sudo yum -y install intel-oneapi-compiler-dpcpp-cpp-2023.1.0.x86_64 +# sudo yum -y install intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2023.1.0.x86_64 +# sudo yum -y install intel-oneapi-compiler-fortran-2023.1.0.x86_64 +# sudo yum -y install --installroot /apps intel-hpckit-2023.1.0.x86_64 + +sudo yum -y install intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2023.1.0.x86_64 +sudo yum -y install intel-oneapi-compiler-fortran-2023.1.0.x86_64 + +mkdir /save/environments/modulefiles + +cd /opt/intel/oneapi/ +./modulefiles-setup.sh --ignore-latest --output-dir=/mnt/efs/fs1/save/environments/modulefiles +module use -a /save/environments/modulefiles + +cd $home + +} + +#-----------------------------------------------------------------------------# + +install_intel_oneapi_spack () { + + echo "Running ${FUNCNAME[0]} ..." + + home=$PWD + + . $SPACK_DIR/share/spack/setup-env.sh + + spack install $SPACKOPTS intel-oneapi-compilers@${ONEAPI_VER} $SPACKTARGET spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin/intel64 spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin - spack install $SPACKOPTS intel-oneapi-mpi@${INTEL_VER} %intel@${INTEL_VER} - spack install $SPACKOPTS intel-oneapi-mkl@${INTEL_VER} %intel@${INTEL_VER} + # MPI will be built with ESMF + # MKL + + # MKL is not installing, a lot of build issues! frustrating! + # sudo yum -y install libxml2 + # sudo yum -y install libxml2-devel + + # Build with Intel Classic compilers + # spack install $SPACKOPTS intel-oneapi-mkl@${ONEAPI_VER} %intel@${INTEL_VER} + # spack install $SPACKOPTS intel-oneapi-mkl@${ONEAPI_VER} ^m4@1.4.18 %intel@${INTEL_VER} $SPACKTARGET + + # >> 1958 /tmp/ec2-user/spack-stage/spack-stage-m4-1.4.17-tw27f45fy4bot6t3an5drrrwdakaewtj/spack-src/lib/fseeko.c(109): error: #error directive: "Please port gnulib fseeko.c to your platform! Look at the code in fseeko.c, then report this to bug-gnulib." + + # Build with Intel OneApi compilers + # spack install $SPACKOPTS intel-oneapi-mkl@${ONEAPI_VER} %oneapi@${ONEAPI_VER} + #spack install $SPACKOPTS intel-oneapi-mkl@${ONEAPI_VER} %oneapi@${ONEAPI_VER} $SPACKTARGET + + # cmp: error while loading shared libraries: libimf.so: cannot open shared object file: No such file or directory + # 3277 /tmp/ec2-user/spack-stage/spack-stage-m4-1.4.19-ty2xeyj2g3cs2jgqukady5zyod4of6eh/spack-src/build-aux/missing: line 81: makeinfo: command not found + # 3278 WARNING: 'makeinfo' is missing on your system. + + # MKL fails with intel classic compiler + # cpx: warning: use of 'dpcpp' is deprecated and will be removed in a future release. Use 'icpx -fsycl' [-Wdeprecated] + # icc: remark #10441: The Intel(R) C++ Compiler Classic (ICC) is deprecated and will be removed from product release in the second half of 2023. The Intel(R) oneAPI DPC++/C++ Compiler (ICX) is the recommended compiler moving forward. Please transition to use this compiler. Use '-diag-disable=10441' to disable this message. cd $home } @@ -285,7 +436,10 @@ install_intel_oneapi () { install_netcdf () { + echo "Running ${FUNCNAME[0]} ..." + echo "Out of date ... returning" + return COMPILER=${COMPILER:-intel@${INTEL_VER}} @@ -316,6 +470,8 @@ install_hdf5-gcc8 () { # This installs the gcc built hdf5 echo "Running ${FUNCNAME[0]} ..." + echo "Out of date ... returning" + return COMPILER=gcc@${GCC_VER} @@ -452,7 +608,7 @@ install_slurm () { spack load intel-oneapi-mpi@${INTEL_VER}%${COMPILER} - #spack install $SPACKOPTS --no-checksum slurm@${SLURM_VER}+hwloc+pmix sysconfdir=/etc/slurm ^tar@1.34%gcc@${GCC_VER} \ + #spack install $SPACKOPTS --no-checksum slurm@${SLURM_VER}+hwloc+pmix sysconfdir=/etc/slurm ^tar@1.34%gcc@${GCC_VER} \ # hwloc build is failing, netloc_mpi_find_hosts.c:116: undefined reference to `MPI_Send' etc. #^"$MUNGEDEP" localstatedir='/var' ^intel-oneapi-mpi@${INTEL_VER} %${COMPILER} @@ -725,29 +881,24 @@ install_slurm-s3() { #-----------------------------------------------------------------------------# -install_esmf () { +install_esmf_spack () { - echo "Running ${FUNCNAME[0]} ..." + # Install esmf along with esmf dependencies such as netcdf and hdf5 - COMPILER=intel@${INTEL_VER} + echo "Running ${FUNCNAME[0]} ..." home=$PWD . $SPACK_DIR/share/spack/setup-env.sh - # ^netcdf-c@4.8.0 - # ^hdf5@1.10.7+cxx+fortran+hl+szip+threadsafe + spack load intel-oneapi-compilers@${ONEAPI_VER} - # spack install $SPACKOPTS esmf%${COMPILER} ^intel-oneapi-mpi@${INTEL_VER} ^diffutils@3.7 ^m4@1.4.17 \ - # ^hdf5@1.10.7+cxx+fortran+hl+szip+threadsafe ^netcdf-c@4.8.0 %${COMPILER} - - # spack install $SPACKOPTS esmf%${COMPILER} ^intel-oneapi-mpi@${INTEL_VER}%gcc@${GCC_VER} ^diffutils@3.7 ^m4@1.4.17 \ - # ^hdf5/qfvg7gc ^netcdf-c/yynmjgt + COMPILER=intel@${INTEL_VER} + spack install $SPACKOPTS esmf@${ESMF_VER} ^intel-oneapi-mpi@${INTEL_VER} %${COMPILER} $SPACKTARGET - spack install $SPACKOPTS esmf%${COMPILER} ^intel-oneapi-mpi@${INTEL_VER} ^diffutils@3.7 ^m4@1.4.17 %${COMPILER} - - # HDF5 also needs szip lib - spack install $SPACKOPTS libszip%${COMPILER} + # Install fails with the following probably because mpi isn't installed with oneapi build + #COMPILER=oneapi@${ONEAPI_VER} + #spack install $SPACKOPTS esmf@${ESMF_VER} ^intel-oneapi-mpi@${INTEL_VER} %${COMPILER} $SPACKTARGET cd $home } @@ -761,6 +912,11 @@ install_base_rpms () { home=$PWD + . /usr/share/Modules/init/bash + + # Only do this once + echo "/usrx/modulefiles" | sudo tee -a ${MODULESHOME}/init/.modulespath + # gcc/6.5.0 hdf5/1.10.5 netcdf/4.5 produtil/1.0.18 esmf/8.0.0 libstar=base_rpms.gcc.6.5.0.el7.20200716.tgz @@ -772,31 +928,26 @@ install_base_rpms () { wget -nv https://ioos-cloud-sandbox.s3.amazonaws.com/public/libs/$libstar tar -xf $libstar rm $libstar - - #rpmlist=' - # hdf5-1.10.5-4.el7.x86_64.rpm - # netcdf-4.5-3.el7.x86_64.rpm - # produtil-1.0.18-2.el7.x86_64.rpm - # esmf-8.0.0-1.el7.x86_64.rpm - #' + sudo yum -y install python2 + sudo alternatives --set python /usr/bin/python2 rpmlist=' produtil-1.0.18-2.el7.x86_64.rpm ' for file in $rpmlist do - sudo yum -y install $file + sudo rpm -ivh $file --nodeps done - rm -Rf "$wrkdir" + # rm -Rf "$wrkdir" cd $home } #-----------------------------------------------------------------------------# -install_extra_rpms () { +install_ncep_rpms () { echo "Running ${FUNCNAME[0]} ..." @@ -835,15 +986,10 @@ install_extra_rpms () { sudo yum -y localinstall $file done - # Force install newer libpng leaving the existing one intact - # this one will be used for our model builds via the module - #sudo rpm -v --install --force libpng-1.5.30-2.el7.x86_64.rpm - # refuses to upgrade # sudo yum -y localinstall libpng-1.5.30-2.el7.x86_64.rpm - # sudo yum -y downgrade libpng-1.5.30-2.el7.x86_64.rpm # WORKS - use spack instead - # rm -Rf "$wrkdir" - sudo yum -y install jasper-devel + #sudo yum -y install jasper-devel + sudo yum -y install jasper-libs cd $home } @@ -856,26 +1002,30 @@ install_python_modules_user () { home=$PWD - sudo python3 -m pip install --upgrade pip - python3 -m pip install --user --upgrade wheel - python3 -m pip install --user --upgrade dask - python3 -m pip install --user --upgrade distributed - python3 -m pip install --user --upgrade setuptools_rust # needed for paramiko - python3 -m pip install --user --upgrade paramiko # needed for dask-ssh - python3 -m pip install --user --upgrade prefect + #python3 -m venv /save/$USER/csvenv + #source /save/$USER/csvenv/bin/activate + + python3 -m pip install pip install prefect==0.15.13 + python3 -m pip install --upgrade pip + python3 -m pip install --upgrade wheel + python3 -m pip install --upgrade dask + python3 -m pip install --upgrade distributed + python3 -m pip install --upgrade setuptools_rust # needed for paramiko + python3 -m pip install --upgrade paramiko # needed for dask-ssh # SPACK has problems with botocore newer than below - python3 -m pip install --user --upgrade botocore==1.23.46 + python3 -m pip install --upgrade botocore==1.23.46 # This is the most recent boto3 that is compatible with botocore above - python3 -m pip install --user --upgrade boto3==1.20.46 + python3 -m pip install --upgrade boto3==1.20.46 # Install requirements for plotting module - cd ../.. + cd ../cloudflow python3 -m pip install --user -r requirements.txt + # install plotting module python3 setup.py sdist - + # deactivate cd $home } @@ -974,30 +1124,90 @@ setup_ssh_mpi () { home=$PWD # MPI needs key to ssh into cluster nodes - sudo -u centos ssh-keygen -t rsa -N "" -C "mpi-ssh-key" -f /home/centos/.ssh/id_rsa - sudo -u centos cat /home/centos/.ssh/id_rsa.pub >> /home/centos/.ssh/authorized_keys + sudo -u $USER ssh-keygen -t rsa -N "" -C "mpi-ssh-key" -f /home/$USER/.ssh/id_rsa + sudo -u $USER cat /home/$USER/.ssh/id_rsa.pub >> /home/$USER/.ssh/authorized_keys -# This - assumes we are using 10.0.x.x subnet addresses +# Prevent SSH prompts when using a local/private address echo " -Host ip-10-0-* + +Host 127.0.0.1 CheckHostIP no - StrictHostKeyChecking no + StrictHostKeyChecking no -Host 10.0.* +Host 10.* CheckHostIP no StrictHostKeyChecking no -" | sudo tee -a /etc/ssh/ssh_config -# cat >> /etc/ssh/ssh_config <> ~/.tcshrc echo alias h history >> ~/.tcshrc - echo alias cds cd /save >> ~/.tcshrc - echo alias cdc cd /com >> ~/.tcshrc - echo alias cdpt cd /ptmp >> ~/.tcshrc + echo alias cds cd /save/$USER >> ~/.tcshrc + echo alias cdc cd /com/$USER >> ~/.tcshrc + echo alias cdpt cd /ptmp/$USER >> ~/.tcshrc #echo alias cdns cd /noscrub >> ~/.tcshrc diff --git a/cloudflow/workflows/scripts/getaws.sh b/scripts/getaws.sh similarity index 100% rename from cloudflow/workflows/scripts/getaws.sh rename to scripts/getaws.sh diff --git a/cloudflow/workflows/scripts/install_conda_jupyterhub.sh b/scripts/install_conda_jupyterhub.sh similarity index 100% rename from cloudflow/workflows/scripts/install_conda_jupyterhub.sh rename to scripts/install_conda_jupyterhub.sh diff --git a/cloudflow/workflows/scripts/install_jupyterhub.sh b/scripts/install_jupyterhub.sh similarity index 100% rename from cloudflow/workflows/scripts/install_jupyterhub.sh rename to scripts/install_jupyterhub.sh diff --git a/cloudflow/workflows/scripts/install_mpi.sh b/scripts/install_mpi.sh similarity index 100% rename from cloudflow/workflows/scripts/install_mpi.sh rename to scripts/install_mpi.sh diff --git a/cloudflow/workflows/scripts/install_prefect.sh b/scripts/install_prefect.sh similarity index 100% rename from cloudflow/workflows/scripts/install_prefect.sh rename to scripts/install_prefect.sh diff --git a/cloudflow/workflows/scripts/install_pythonmods.sh b/scripts/install_pythonmods.sh similarity index 100% rename from cloudflow/workflows/scripts/install_pythonmods.sh rename to scripts/install_pythonmods.sh diff --git a/cloudflow/workflows/scripts/install_slurm.sh b/scripts/install_slurm.sh similarity index 100% rename from cloudflow/workflows/scripts/install_slurm.sh rename to scripts/install_slurm.sh diff --git a/cloudflow/workflows/scripts/mountEFS.sh b/scripts/mountEFS.sh similarity index 100% rename from cloudflow/workflows/scripts/mountEFS.sh rename to scripts/mountEFS.sh diff --git a/cloudflow/workflows/scripts/ncdis-setup.sh b/scripts/ncdis-setup.sh similarity index 100% rename from cloudflow/workflows/scripts/ncdis-setup.sh rename to scripts/ncdis-setup.sh diff --git a/cloudflow/workflows/scripts/old-setup-jupyterhub.sh b/scripts/old-setup-jupyterhub.sh similarity index 100% rename from cloudflow/workflows/scripts/old-setup-jupyterhub.sh rename to scripts/old-setup-jupyterhub.sh diff --git a/cloudflow/workflows/scripts/osx-setup-local.sh b/scripts/osx-setup-local.sh similarity index 100% rename from cloudflow/workflows/scripts/osx-setup-local.sh rename to scripts/osx-setup-local.sh diff --git a/scripts/resume-setup.sh b/scripts/resume-setup.sh new file mode 100755 index 0000000..4798a03 --- /dev/null +++ b/scripts/resume-setup.sh @@ -0,0 +1,15 @@ +#!/bin/env bash +# set -x +#scl enable gcc-toolset-11 bash +source scl_source enable gcc-toolset-11 + +cd /save/environments/spack-stack +source setup.sh +cd envs/ioos-aws-rhel + +module use -a /save/environments/modulefiles + +module load compiler +module load icc + +spack env activate -p . diff --git a/scripts/scp-secret.sh b/scripts/scp-secret.sh new file mode 100755 index 0000000..349cfab --- /dev/null +++ b/scripts/scp-secret.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +# ssh -i ~/.ssh/ioos-sandbox.pem ec2-user@ec2-3-128-236-96.us-east-2.compute.amazonaws.com +host='ec2-3-130-117-202.us-east-2.compute.amazonaws.com' + +user='ec2-user' + +scp -p -i ~/.ssh/ioos-sandbox.pem ~/OD/SPACK/spack.mirror.gpgkey.secret ${user}@${host}:/save/environments/spack/opt/spack/gpg + +ssh -i ~/.ssh/ioos-sandbox.pem ${user}@${host} 'cd /save/environments/spack/opt/spack/gpg; spack gpg trust spack.mirror.gpgkey.secret' diff --git a/cloudflow/workflows/scripts/script.rpm.git-lfs.sh b/scripts/script.rpm.git-lfs.sh similarity index 100% rename from cloudflow/workflows/scripts/script.rpm.git-lfs.sh rename to scripts/script.rpm.git-lfs.sh diff --git a/cloudflow/workflows/scripts/scrub_liveocean.sh b/scripts/scrub_liveocean.sh similarity index 100% rename from cloudflow/workflows/scripts/scrub_liveocean.sh rename to scripts/scrub_liveocean.sh diff --git a/cloudflow/workflows/scripts/scrub_nosofs.sh b/scripts/scrub_nosofs.sh similarity index 100% rename from cloudflow/workflows/scripts/scrub_nosofs.sh rename to scripts/scrub_nosofs.sh diff --git a/scripts/setup-instance.sh b/scripts/setup-instance.sh new file mode 100755 index 0000000..9519530 --- /dev/null +++ b/scripts/setup-instance.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +#__copyright__ = "Copyright © 2023 RPS Group, Inc. All rights reserved." +#__license__ = "BSD 3-Clause" + +GCC_VER=11.2.1 + +# Current versions +ONEAPI_VER=2023.1.0 + +# There is no oneapi mpi version 2023.1.0 +INTEL_VER=2021.9.0 +# MPI_VER=2021.9.0 +ESMF_VER=8.5.0 + +#SPACK_VER='releases/v0.18' +#SPACK_DIR='/save/environments/spack-stack/spack' + +SPACK_VER='v0.21.0' +SPACK_DIR='/save/environments/spack' + +SPACKOPTS='-v -y' + +#SPACKTARGET='target=skylake_avx512' # default on skylake intel instances t3.xxxx +#export SPACKTARGET='target=haswell' # works on AMD also - has no avx512 extensions +#SPACKTARGET='target=x86_64' # works on anything +SPACKTARGET="arch=linux-rhel8-x86_64" + +# 1 = Don't build any packages. Only install packages from binary mirrors +# 0 = Will build if not found in mirror/cache +# -1 = Don't check pre-built binary cache +SPACK_CACHEONLY=1 + +########################################################## + +# source include the functions +. funcs-setup-instance.sh + +# calling sudo from cloud init adds 25 second delay for each sudo command +sudo setenforce 0 + +# Use caution when changing the order of the following + +# System stuff +setup_paths +setup_aliases +setup_environment + +module use -a /save/environments/modulefiles + +## install_jupyterhub # Requires some manual work +setup_ssh_mpi +install_efa_driver + +# Compilers and libraries +install_python_modules_user +install_gcc_toolset_yum + +source /opt/rh/gcc-toolset-11/enable + +install_spack +install_intel_oneapi_spack +install_esmf_spack + +install_base_rpms +install_ncep_rpms + +# install_ffmpeg + +# TODO: create an output file to contain all of this state info - json +# TODO: re-write in Python ? + +# create node image +################################### + +# ami_name is provided by Terraform if called via the init_template +# otherwise it will use the default +ami_name=${ami_name:='IOOS-Cloud-Sandbox'} + +# TODO: pass this in via Terraform init template +project_tag=${project_tag:="IOOS-Cloud-Sandbox"} + +# create node image +################################### + +./create_image.sh $ami_name $project_tag + +echo "Setup completed!" diff --git a/scripts/setup-python-deps.sh b/scripts/setup-python-deps.sh new file mode 100755 index 0000000..b129ef6 --- /dev/null +++ b/scripts/setup-python-deps.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# source include the functions +. funcs-setup-instance.sh + +# Compilers and libraries +install_python_modules_user diff --git a/scripts/spack-stack-setup.sh b/scripts/spack-stack-setup.sh new file mode 100755 index 0000000..47f8231 --- /dev/null +++ b/scripts/spack-stack-setup.sh @@ -0,0 +1,105 @@ +#!/bin/env bash + +home=$PWD + +# Compilers - this includes environment module support +#PT sudo yum -y install gcc-toolset-11-gcc-c++ +#PT sudo yum -y install gcc-toolset-11-gcc-gfortran +#PT sudo yum -y install gcc-toolset-11-gdb + +# Misc +#PT sudo yum -y install m4 +#PT sudo yum -y install wget +#PT sudo yum -y install git +#PT sudo yum -y install git-lfs +#PT sudo yum -y install bash-completion +#PT sudo yum -y install bzip2 bzip2-devel +#PT sudo yum -y install unzip +#PT sudo yum -y install patch +#PT sudo yum -y install automake +#PT sudo yum -y install xorg-x11-xauth +#PT sudo yum -y install xterm +#PT sudo yum -y install texlive +#PT sudo yum -y install mysql-server + +# Do not install cmake (it's 3.20.2, which doesn't work with eckit) +# Do not install qt@5 for now + +# For screen utility (optional) +#yum -y remove https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm +#yum -y update --nobest +#yum -y install screen + +# Python +#PT sudo yum -y install python3.11-devel +#PT sudo alternatives --set python3 /usr/bin/python3.11 +#PT sudo yum -y install python3.11-pip + +# Install intel oneAPI compilers via yum - Tried that, didnt work so well + +#PT source scl_source enable gcc-toolset-11 + +cd /save/environments +# git clone --recurse-submodules -b ioos-aws https://github.com/asascience/spack-stack.git +cd spack-stack +source setup.sh + +#export SPACK_STACK_DIR +#echo "Setting environment variable SPACK_STACK_DIR to ${SPACK_STACK_DIR}" + +#source ${SPACK_STACK_DIR}/spack/share/spack/setup-env.sh +#echo "Sourcing spack environment ${SPACK_STACK_DIR}/spack/share/spack/setup-env.sh" + +# Creating a new site config +ENVNAME="ioos-aws-rhel8-x86-64" +#PT spack stack create env --name $ENVNAME +cd envs/$ENVNAME/ +spack env activate -p . + +#PT export SPACK_SYSTEM_CONFIG_PATH="$PWD" + +#PT spack external find --scope system --exclude bison --exclude cmake --exclude curl --exclude openssl --exclude openssh +#PT spack external find --scope system perl +#PT spack external find --scope system wget +#PT spack external find --scope system mysql +#PT spack external find --scope system texlive +#PT cat >> packages.yaml <&1 | tee log.concretize +#PT util/show_duplicate_packages.py -d -c log.concretize + +#PT spack install --verbose --fail-fast + +#PT spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin/intel64 +#PT spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin + +#PT spack module tcl refresh +#PT spack stack setup-meta-modules + +# Add more packages and repeat + +# spack add base-env arch=linux-rhel8-x86_64 + +exit + +# Create meta-modules for compiler, mpi, python + diff --git a/cloudflow/workflows/scripts/spack-update-mirror.sh b/scripts/spack-update-mirror.sh similarity index 50% rename from cloudflow/workflows/scripts/spack-update-mirror.sh rename to scripts/spack-update-mirror.sh index b26846c..9d908d2 100755 --- a/cloudflow/workflows/scripts/spack-update-mirror.sh +++ b/scripts/spack-update-mirror.sh @@ -1,19 +1,24 @@ #!/usr/bin/env bash SPACK_MIRROR='s3://ioos-cloud-sandbox/public/spack/mirror' +JOBS=4 #SPEC='%gcc@4.8' #SPEC=%intel@2021.3.0 #SPEC='%gcc@8.5' -SPEC_LIST='%intel@2021.3.0 %gcc@8.5 gcc@4.85' -### MAKE SURE TO IMPORT THE PRIVATE KEY FIRST! -#SECRET=~/spack.mirror.gpgkey.secret +#SPEC_LIST='%dpcpp@2023.1.0 %gcc@11.2.1 %intel@2021.9.0 %oneapi@2023.1.0' +SPEC_LIST='%gcc@11.2.1 %intel@2021.9.0' + +### MAKESURE TO IMPORT THE PRIVATE KEY FIRST! +#SECRET=/mnt/efs/fs1/save/environments/spack/opt/spack/gpg/spack.mirror.gpgkey.secret #spack gpg trust $SECRET #exit +KEY=F525C05B06DCA266 + # Rebuild PKGLIST=`spack find --format "{name}@{version}%{compiler}/{hash}" $SPEC` @@ -23,6 +28,6 @@ do for PKG in $PKGLIST do echo "PACKAGE: $PKG" - spack buildcache create --rebuild-index -a -r --mirror-url $SPACK_MIRROR $PKG + spack buildcache push -k $KEY -j $JOBS --only package $SPACK_MIRROR $PKG done done diff --git a/cloudflow/workflows/scripts/system/default.conf b/scripts/system/default.conf similarity index 100% rename from cloudflow/workflows/scripts/system/default.conf rename to scripts/system/default.conf diff --git a/cloudflow/workflows/scripts/system/git.munge.systemd.service.in b/scripts/system/git.munge.systemd.service.in similarity index 100% rename from cloudflow/workflows/scripts/system/git.munge.systemd.service.in rename to scripts/system/git.munge.systemd.service.in diff --git a/cloudflow/workflows/scripts/system/github.slurmctld.service.in b/scripts/system/github.slurmctld.service.in similarity index 100% rename from cloudflow/workflows/scripts/system/github.slurmctld.service.in rename to scripts/system/github.slurmctld.service.in diff --git a/cloudflow/workflows/scripts/system/github.slurmd.service.in b/scripts/system/github.slurmd.service.in similarity index 100% rename from cloudflow/workflows/scripts/system/github.slurmd.service.in rename to scripts/system/github.slurmd.service.in diff --git a/cloudflow/workflows/scripts/system/jupyterhub.conf b/scripts/system/jupyterhub.conf similarity index 100% rename from cloudflow/workflows/scripts/system/jupyterhub.conf rename to scripts/system/jupyterhub.conf diff --git a/cloudflow/workflows/scripts/system/jupyterhub.service b/scripts/system/jupyterhub.service similarity index 100% rename from cloudflow/workflows/scripts/system/jupyterhub.service rename to scripts/system/jupyterhub.service diff --git a/cloudflow/workflows/scripts/system/jupyterhub_config.py b/scripts/system/jupyterhub_config.py similarity index 100% rename from cloudflow/workflows/scripts/system/jupyterhub_config.py rename to scripts/system/jupyterhub_config.py diff --git a/cloudflow/workflows/scripts/system/jupytersudoers b/scripts/system/jupytersudoers similarity index 100% rename from cloudflow/workflows/scripts/system/jupytersudoers rename to scripts/system/jupytersudoers diff --git a/cloudflow/workflows/scripts/system/munge.service b/scripts/system/munge.service similarity index 100% rename from cloudflow/workflows/scripts/system/munge.service rename to scripts/system/munge.service diff --git a/cloudflow/workflows/scripts/system/munge.service.in b/scripts/system/munge.service.in similarity index 100% rename from cloudflow/workflows/scripts/system/munge.service.in rename to scripts/system/munge.service.in diff --git a/cloudflow/workflows/scripts/system/nginx.conf b/scripts/system/nginx.conf similarity index 100% rename from cloudflow/workflows/scripts/system/nginx.conf rename to scripts/system/nginx.conf diff --git a/cloudflow/workflows/scripts/system/runJupyterhub.sh b/scripts/system/runJupyterhub.sh similarity index 100% rename from cloudflow/workflows/scripts/system/runJupyterhub.sh rename to scripts/system/runJupyterhub.sh diff --git a/cloudflow/workflows/scripts/system/slurm.conf b/scripts/system/slurm.conf similarity index 100% rename from cloudflow/workflows/scripts/system/slurm.conf rename to scripts/system/slurm.conf diff --git a/cloudflow/workflows/scripts/system/slurmctld.service b/scripts/system/slurmctld.service similarity index 100% rename from cloudflow/workflows/scripts/system/slurmctld.service rename to scripts/system/slurmctld.service diff --git a/cloudflow/workflows/scripts/system/slurmctld.service.in b/scripts/system/slurmctld.service.in similarity index 100% rename from cloudflow/workflows/scripts/system/slurmctld.service.in rename to scripts/system/slurmctld.service.in diff --git a/cloudflow/workflows/scripts/system/slurmd.service b/scripts/system/slurmd.service similarity index 100% rename from cloudflow/workflows/scripts/system/slurmd.service rename to scripts/system/slurmd.service diff --git a/cloudflow/workflows/scripts/system/slurmd.service.in b/scripts/system/slurmd.service.in similarity index 100% rename from cloudflow/workflows/scripts/system/slurmd.service.in rename to scripts/system/slurmd.service.in diff --git a/cloudflow/workflows/scripts/system/slurmdbd.service b/scripts/system/slurmdbd.service similarity index 100% rename from cloudflow/workflows/scripts/system/slurmdbd.service rename to scripts/system/slurmdbd.service diff --git a/scripts/test-setup.sh b/scripts/test-setup.sh new file mode 100755 index 0000000..628c7f0 --- /dev/null +++ b/scripts/test-setup.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash + +#__copyright__ = "Copyright © 2023 RPS Group, Inc. All rights reserved." +#__license__ = "BSD 3-Clause" + +GCC_VER=11.2.1 + +# Current versions +ONEAPI_VER=2023.1.0 + +# There is no oneapi mpi version 2023.1.0 +INTEL_VER=2021.9.0 +# MPI_VER=2021.9.0 +ESMF_VER=8.5.0 + +#SPACK_VER='releases/v0.18' +#SPACK_DIR='/save/environments/spack-stack/spack' + +SPACK_VER='v0.21.0' +SPACK_DIR='/save/environments/spack' +#sudo mkdir -p $SPACK_DIR +#sudo chown ec2-user:ec2-user $SPACK_DIR + +SPACKOPTS='-v -y' + +# SPACKTARGET is only used for some model libraries such as MPI, +#SPACKTARGET='target=skylake_avx512' # default on skylake intel instances t3.xxxx +#export SPACKTARGET='target=haswell' # works on AMD also - has no avx512 extensions +#SPACKTARGET='target=x86_64' # works on anything +SPACKTARGET="arch=linux-rhel8-x86_64" + + +# 1 = Don't build any packages. Only install packages from binary mirrors +# 0 = Will build if not found in mirror/cache +# -1 = Don't check pre-built binary cache +SPACK_CACHEONLY=0 + + +########################################################## + + +# source include the functions +. funcs-setup-instance.sh + +# calling sudo from cloud init adds 25 second delay for each sudo command +sudo setenforce 0 + +# Use caution when changing the order of the following + +# System stuff +#setup_paths +#setup_aliases +#setup_environment + +module use -a /save/environments/modulefiles + +## install_jupyterhub # Requires some manual work +#setup_ssh_mpi +#install_efa_driver + +# Compilers and libraries +#install_python_modules_user +#install_gcc_toolset_yum + +source /opt/rh/gcc-toolset-11/enable + +#install_spack +# install_intel_oneapi_spack + +#install_esmf_spack + +#install_base_rpms +# install_ncep_rpms +# install_ffmpeg + +# TODO: create an output file to contain all of this state info - json +# TODO: re-write in Python ? + +# create node image +################################### + +# ami_name is provided by Terraform if called via the init_template +# otherwise it will use the default +ami_name=${ami_name:='IOOS-Cloud-Sandbox'} + +# TODO: pass this in via Terraform init template +project_tag=${project_tag:="IOOS-Cloud-Sandbox"} + +# create node image +################################### + +#./create_image.sh $ami_name $project_tag + +echo "Setup completed!" diff --git a/cloudflow/workflows/scripts/test_user.sh b/scripts/test_user.sh similarity index 100% rename from cloudflow/workflows/scripts/test_user.sh rename to scripts/test_user.sh diff --git a/setup-plotting.sh b/setup-plotting.sh new file mode 100755 index 0000000..f3e9dbd --- /dev/null +++ b/setup-plotting.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +python3 -m pip install . +python3 -m pip install ./cloudflow +mkdir /com/nos +touch /com/nos/current.fcst + + diff --git a/setup.py b/setup.py index 146cd79..8aff1de 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ # setup cloud-workflow package setup(name="cloudflow", - version='1.3.2', + version='1.6.0b', description='Workflows for cloud based numerical weather prediction', url='https://github.com/asascience/Cloud-Sandbox', author='RPS North America', @@ -33,6 +33,6 @@ 'matplotlib', 'netCDF4', 'numpy', - 'pyproj==2.6', + 'pyproj', 'plotting'] ) diff --git a/terraform/TODO.txt b/terraform/TODO.txt index fd06e4e..0058c5a 100644 --- a/terraform/TODO.txt +++ b/terraform/TODO.txt @@ -1,3 +1,5 @@ Add the clear display/output of the following for cloudflow cluster config files: Eventually this should be automated - abstracted away from the user view. +change security group to allow multiple ssh cidrs + diff --git a/terraform/config.s3.tfbackend b/terraform/config.s3.tfbackend index c8c1f4c..ab83374 100644 --- a/terraform/config.s3.tfbackend +++ b/terraform/config.s3.tfbackend @@ -1,2 +1,2 @@ bucket = "cloud-sandbox-tfstate" -region = "us-east-1" +region = "us-east-2" diff --git a/terraform/init_template.tpl b/terraform/init_template.tpl index f773125..2a5e3ba 100644 --- a/terraform/init_template.tpl +++ b/terraform/init_template.tpl @@ -1,11 +1,15 @@ #!/usr/bin/env bash +set -x echo `date` > /tmp/setup.log -# AMZ linux and some other AMIs use ec2-user. CentOS 7 usese centos +# RHEL8+ +RUNUSER="ec2-user" +#BRANCH=origin/x86_64 +BRANCH=main -# RUNUSER="ec2-user" -RUNUSER="centos" +# CentOS 7 - Stream 8 +#RUNUSER="centos" # Mount the EFS volume @@ -32,15 +36,22 @@ mount -t nfs4 "${efs_name}:/" /mnt/efs/fs1 echo "${efs_name}:/ /mnt/efs/fs1 nfs defaults,_netdev 0 0" >> /etc/fstab cd /mnt/efs/fs1 -sudo mkdir save -sudo chgrp wheel save -sudo chmod 777 save +if [ ! -d save ] ; then + sudo mkdir save + sudo chgrp wheel save + sudo chmod 777 save + sudo ln -s /mnt/efs/fs1/save /save +fi -# Clone the Cloud-Sandbox repository # Placing this in a common location cd /mnt/efs/fs1/save +sudo mkdir $RUNUSER +sudo chown $RUNUSER:$RUNUSER $RUNUSER +cd $RUNUSER sudo -u $RUNUSER git clone https://github.com/ioos/Cloud-Sandbox.git -cd Cloud-Sandbox/cloudflow/workflows/scripts +cd Cloud-Sandbox +sudo -u $RUNUSER git checkout -t $BRANCH +cd scripts # Need to pass ami_name export ami_name=${ami_name} diff --git a/terraform/login b/terraform/login new file mode 100644 index 0000000..34a040a --- /dev/null +++ b/terraform/login @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +$(terraform output | grep login_command | awk -F= '{print $2}' | awk -F\" '{print $2}') diff --git a/terraform/main.tf b/terraform/main.tf index e23494c..d5267ce 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -15,7 +15,7 @@ provider "aws" { } resource "aws_iam_role" "sandbox_iam_role" { - name = "${var.nameprefix}_terraform_role" + name = "${var.nameprefix}-${var.availability_zone}_terraform_role" assume_role_policy = jsonencode( { "Version" : "2012-10-17", @@ -43,12 +43,13 @@ resource "aws_iam_role_policy_attachment" "sandbox_role_policy_attach" { } resource "aws_iam_instance_profile" "cloud_sandbox_iam_instance_profile" { - name = "${var.nameprefix}_terraform_role" + name = "${var.nameprefix}-${var.availability_zone}_terraform_instance_profile" role = aws_iam_role.sandbox_iam_role.name } + resource "aws_placement_group" "cloud_sandbox_placement_group" { - name = "${var.nameprefix}_Terraform_Placement_Group" + name = "${var.nameprefix}-${var.availability_zone}_Terraform_Placement_Group" strategy = "cluster" tags = { project = var.project_tag @@ -111,6 +112,7 @@ locals { resource "aws_internet_gateway" "gw" { + count = var.subnet_id != null ? 0 : 1 vpc_id = local.vpc.id tags = { Name = "${var.name_tag} Internet Gateway" @@ -119,6 +121,7 @@ resource "aws_internet_gateway" "gw" { } resource "aws_route_table" "default" { + count = var.subnet_id != null ? 0 : 1 vpc_id = local.vpc.id route { @@ -132,6 +135,7 @@ resource "aws_route_table" "default" { } resource "aws_route_table_association" "main" { + count = var.subnet_id != null ? 0 : 1 subnet_id = one(aws_subnet.main[*].id) route_table_id = one(aws_route_table.default[*].id) } @@ -146,6 +150,7 @@ resource "aws_efs_file_system" "main_efs" { } } + resource "aws_efs_mount_target" "mount_target_main_efs" { subnet_id = local.subnet.id security_groups = [aws_security_group.efs_sg.id] @@ -156,18 +161,21 @@ resource "aws_efs_mount_target" "mount_target_main_efs" { ### AMI Options ### Specify aws_ami.*****.id in aws_instance section below -######################## -# CentOS 7 CentOS AMI -# ami-033adaf0b583374d4 -######################## -data "aws_ami" "centos_7" { - owners = ["125523088429"] # CentOS.org +################################### +# RHEL 8 +# 2023-10-06: ami-057094267c651958e +# AMI name: RHEL-8.7.0_HVM-20230330-x86_64-56-Hourly2-GP2 +# Owner account ID 309956199498 Red Hat +################################## + +data "aws_ami" "rhel_8" { + owners = ["309956199498"] most_recent = true filter { name = "name" - values = ["CentOS Linux 7 *"] + values = ["RHEL-8.7.0_HVM-20230330-x86_64-56-Hourly2-GP2"] } filter { @@ -187,91 +195,20 @@ data "aws_ami" "centos_7" { } -######################## -# CentOS 7 AMI by AWS - -# 2023-04-17 us-east-2 id -# ami-05a36e1502605b4aa -######################## - -data "aws_ami" "centos_7_aws" { - owners = ["679593333241"] # AWS Marketplace - most_recent = true - - filter { - name = "description" - values = ["CentOS-7*"] - } - - filter { - name = "architecture" - values = ["x86_64"] - } - - filter { - name = "root-device-type" - values = ["ebs"] - } - - filter { - name = "virtualization-type" - values = ["hvm"] - } -} - -################################### -# Amazon Linux 2 AMI - untested -# ami-0b0f111b5dcb2800f Amazon Linux 2 Kernel 5.10 AMI 2.0.20230404.1 x86_64 HVM gp2 -# ami-02751969195641ff2 Amazon Linux 2 SELinux Enforcing AMI 2.0.20230404.1 x86_64 Minimal HVM gp2 ################################### - -data "aws_ami" "amazon_linux_2" { - #owners = ["679593333241"] # AWS Marketplace - #owners = ["137112412989"] # amazon - owners = ["amazon"] # AWS - most_recent = true - - filter { - name = "description" - values = ["Amazon Linux 2 Kernel 5.*"] - } - - filter { - name = "architecture" - values = ["x86_64"] - } - - filter { - name = "root-device-type" - values = ["ebs"] - } - - filter { - name = "virtualization-type" - values = ["hvm"] - } -} - -################################### -# Amazon Linux 2023 AMI - untested +# Centos Stream 9 - untested +# ami-0c2abda83f1b9e09d +# AMI name: CentOS Stream 9 x86_64 +# Owner account ID 125523088429 ################################## -# id us-east-2 2023-04-17 Description -# ----------------------- ----------- -# ami-0103f211a154d64a6 "Amazon Linux 2023 AMI 2023.0.20230329.0 x86_64 HVM kernel-6.1" -# ami-00d80f7cbbd22eb22 "Amazon Linux 2023 AMI 2023.0.20230329.0 x86_64 Minimal HVM kernel-6.1" -################################### - -data "aws_ami" "amazon_linux_2023" { - #owners = ["679593333241"] # AWS Marketplace - #owners = ["137112412989"] # amazon - owners = ["amazon"] # AWS +data "aws_ami" "centos_stream_9" { + owners = ["125523088429"] # CentOS Official CPE most_recent = true filter { name = "description" - values = ["Amazon Linux 2023 * x86_64 HVM *"] - # values = ["Amazon Linux 2023 * x86_64 Minimal HVM *"] + values = ["CentOS Stream 9 *"] } filter { @@ -291,34 +228,9 @@ data "aws_ami" "amazon_linux_2023" { } -################################ -# RedHat 8 UFS AMI - testing -################################ - -data "aws_ami" "rh_ufs" { - owners = ["309956199498"] # NOAA user - most_recent = true - - filter { - name = "name" - values = ["RHEL-8.4.*x86_64*"] - #values = ["RHEL-8.2.0_HVM-20210907-x86_64-0-Hourly2-GP2"] # openSSL yum issues - } - - filter { - name = "root-device-type" - values = ["ebs"] - } - - filter { - name = "virtualization-type" - values = ["hvm"] - } -} - - # Work around to get a public IP assigned when using EFA resource "aws_eip" "head_node" { + count = var.subnet_id != null ? 0 : 1 depends_on = [aws_internet_gateway.gw] vpc = true instance = aws_instance.head_node.id @@ -331,46 +243,48 @@ resource "aws_eip" "head_node" { # efa enabled node resource "aws_instance" "head_node" { - # Base CentOS 7 AMI, can use either AWS's marketplace, or direct from CentOS - # Choosing direct from CentOS as it is more recent - ################################# ### Specify which AMI to use here - ### Only CentOS 7 has been thoroughly tested ############################################# - ami = data.aws_ami.centos_7.id - # ami = data.aws_ami.centos_7_aws.id - - # Untested - # ami = data.aws_ami.amazon_linux_2.id - # ami = data.aws_ami.amazon_linux_2023.id + ami = data.aws_ami.rhel_8.id - # Can optionally use redhat - use the parameterized - # ami = data.aws_ami.rh_ufs.id metadata_options { - http_tokens = "required" + http_endpoint = "enabled" + http_tokens = "required" } + instance_type = var.instance_type cpu_threads_per_core = 2 + root_block_device { encrypted = true delete_on_termination = true - volume_size = 12 + volume_size = 16 + volume_type = "gp3" + tags = { + Name = "${var.name_tag} Head Node" + Project = var.project_tag + } + } depends_on = [aws_internet_gateway.gw, - aws_efs_file_system.main_efs, - aws_efs_mount_target.mount_target_main_efs] + aws_efs_file_system.main_efs, + aws_efs_mount_target.mount_target_main_efs] key_name = var.key_name iam_instance_profile = aws_iam_instance_profile.cloud_sandbox_iam_instance_profile.name + + # user_data = data.template_file.init_instance.rendered + user_data = templatefile("init_template.tpl", { efs_name = aws_efs_file_system.main_efs.dns_name, ami_name = "${var.name_tag}-${random_pet.ami_id.id}", aws_region = var.preferred_region, project = var.project_tag }) # associate_public_ip_address = true network_interface { network_interface_id = aws_network_interface.head_node.id + device_index = 0 } # This logic isn't perfect since some ena instance types can be in a placement group also @@ -392,18 +306,6 @@ resource "random_pet" "ami_id" { length = 2 } -#data "template_file" "init_instance" { -# template = file("./init_template.tpl") -# vars = { -# efs_name = aws_efs_file_system.main_efs.dns_name -# ami_name = "${var.name_tag}-${random_pet.ami_id.id}" -# aws_region = var.preferred_region -# project = var.project_tag -# } -# -#depends_on = [aws_efs_file_system.main_efs, -# aws_efs_mount_target.mount_target_main_efs] -#} # Can only attach efa adaptor to a stopped instance! resource "aws_network_interface" "head_node" { diff --git a/terraform/mysettings.tfvars b/terraform/mysettings.tfvars index 56cfc46..9534476 100644 --- a/terraform/mysettings.tfvars +++ b/terraform/mysettings.tfvars @@ -9,17 +9,18 @@ # The following variables must be defined, no default exists: #------------------------------------------------------------ -# (This should be the public IPv4 address of your workstation for SSH acceess) -# This should be in the format ###.###.###.###/32 -# Example: 72.245.67.89/32 +# List of the IP's address the are allowed SSH access to the system +# They should be in the format ###.###.###.###/32 +# Example: +# allowed_ssh_cidr_list = ["72.200.162.64/32", "96.238.4.28/32", "96.238.4.30/32"] -allowed_ssh_cidr = "" +allowed_ssh_cidr_list = ["", "", ""] # Specify the SSL key below. # Example: # key_name = "ioos-sandbox" -# public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2cQ3fzP1R2uQJcCd3g2ylW5clzyjun6eWz2PZKMwtJh7E28B1jp3F8YTP5XBPg0ouvZO6gkcrbgjhuM0A4NKJM6RylGAOqqPYnIbhd9eI3RKhSQbxsghjf5hwS7tIG1FebO9HuObaM23LDB1/Ra/YMTXB5LHPChlfxrEIlM/7OUfRPNgtudAb/MQZ+YD+6I77QDtTwZwQvebxLK62bP5CrpV4XY5ybWOZ0T3m4pVNfhfl7+QWAvWeStNpH3B3q1ZtPLTuAVvsR4RWk7t75IwpHwiPBcgZn/PTpN45z" +# public_key = "ssh-rsa AANzaC2AADAQABAAABAQC2cQ3fzP1R2uQJcCd3g2ylW5clzyjun6eWz2PZKMwtJh7E28B1jp3F8YTP5XBPg0ouvZO6gkcrbgjhuM0A4NKJM6RylGAOqqPYnIbhd9eI3RKhSQbxsghjf5hwS7tIG1FebO9HuObaM23LDBQvebxLK62bP5CrpV4XY5ybWOZ0T3m4pVNfhfl71ZtPLTuAVvsR4RWk7t75IwpHwiPBcgZn/PTpN45z" key_name = "" # the filename of the private SSL key public_key = "" # the matching public key (ssh-keygen -y -f your-key-pair.pem) @@ -27,27 +28,28 @@ public_key = "" # the matching public key (ssh-keygen -y -f your-key-pair.pem) # Example: # vpc_id = "vpc-0381e9f82c9ae68e7" -vpc_id = "" # the ID of an existing VPC to deploy resources to +# vpc_id = "" # the ID of an existing VPC to deploy resources to # Example: # subnet_id = "subnet-01ce99f9006e8ed06" -subnet_id = "" # the ID of an existing Subnet within the VPC to deploy resources to - +# subnet_id = "" # the ID of an existing Subnet within the VPC to deploy resources to +# Must specify this when not using a pre-provisioned subnet: +subnet_cidr = "10.0.0.0/24" #------------------------------------------------------------ #--------------------------------------------------------------- -# Optionally uncomment and change these to override the defaults +# Optionally uncomment and change the defaults shown #--------------------------------------------------------------- # preferred_region = "us-east-2" -# name_tag = "IOOS-Cloud-Sandbox-Terraform" -# availability_zone = "us-east-2a" +# name_tag = "IOOS-Cloud-Sandbox" +# availability_zone = "us-east-2b" # project_tag = "IOOS-Cloud-Sandbox" -# instance_type = "t3.medium" +# instance_type = "t3.xlarge" # use_efa = false # You can give your AWS resources a unique name to avoid conflicts diff --git a/terraform/outputs.tf b/terraform/outputs.tf index e6c8221..bb00244 100644 --- a/terraform/outputs.tf +++ b/terraform/outputs.tf @@ -1,16 +1,16 @@ output "instance_id" { description = "The EC2 instance id" - value = aws_instance.head_node.id + value = one(aws_instance.head_node[*]).id } output "instance_public_ip" { description = "Public IP Address of the EC2 Instance" - value = aws_eip.head_node.public_ip + value = one(aws_eip.head_node[*]) != null ? one(aws_eip.head_node[*]).public_ip : null } output "instance_public_dns" { description = "Public DNS Address of the EC2 Instance" - value = aws_eip.head_node.public_dns + value = one(aws_eip.head_node[*]) != null ? one(aws_eip.head_node[*]).public_dns : null } output "key_name" { @@ -66,12 +66,7 @@ output "aws_placement_group" { } output "login_command" { - description = "SSH Login" - value = "ssh -i ${var.key_name}.pem centos@${aws_eip.head_node.public_dns}" + description = "SSH Login" + value = one(aws_eip.head_node[*]) != null ? "ssh -i ~/.ssh/${var.key_name}.pem ec2-user@${one(aws_eip.head_node[*]).public_dns}" : "ssh -i ~/.ssh/${var.key_name}.pem ec2-user@${aws_instance.head_node.private_dns}" } -# output "login_command" { -# description = "SSH Login" -# value = "ssh -i ${var.key_name}.pem ec2-user@${aws_eip.head_node.public_dns}" -#} - diff --git a/terraform/pipeline.tf b/terraform/pipeline/pipeline.tf old mode 100644 new mode 100755 similarity index 100% rename from terraform/pipeline.tf rename to terraform/pipeline/pipeline.tf diff --git a/terraform/remote-state/s3.defaults.tfvars b/terraform/remote-state/s3.defaults.tfvars index d4e9fe3..a4c9129 100644 --- a/terraform/remote-state/s3.defaults.tfvars +++ b/terraform/remote-state/s3.defaults.tfvars @@ -1,4 +1,4 @@ # The default values for S3 bucket creation that can be passed to terraform init via '-var-file=s3.defaults.tfvars' flag (or modified and then passed) tfstate_s3_bucket_name = "cloud-sandbox-tfstate" -preferred_region = "us-east-1" +preferred_region = "us-east-2" diff --git a/terraform/save/main.tf.saved b/terraform/save/main.tf.saved new file mode 100644 index 0000000..1d86410 --- /dev/null +++ b/terraform/save/main.tf.saved @@ -0,0 +1,400 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 3.47" + } + } + backend "s3" { + key = "tfstate" + } +} + +provider "aws" { + region = var.preferred_region +} + +resource "aws_iam_role" "sandbox_iam_role" { + name = "${var.nameprefix}_terraform_role" + assume_role_policy = jsonencode( + { + "Version" : "2012-10-17", + "Statement" : [ + { + "Action" : "sts:AssumeRole", + "Principal" : { + "Service" : "ec2.amazonaws.com" + }, + "Effect" : "Allow", + "Sid" : "" + } + ] + }) + tags = { + Name = "${var.name_tag} IAM Role" + Project = var.project_tag + } +} + +resource "aws_iam_role_policy_attachment" "sandbox_role_policy_attach" { + count = length(var.managed_policies) + policy_arn = element(var.managed_policies, count.index) + role = aws_iam_role.sandbox_iam_role.name +} + +resource "aws_iam_instance_profile" "cloud_sandbox_iam_instance_profile" { + name = "${var.nameprefix}_terraform_role" + role = aws_iam_role.sandbox_iam_role.name +} + +resource "aws_placement_group" "cloud_sandbox_placement_group" { + name = "${var.nameprefix}_Terraform_Placement_Group" + strategy = "cluster" + tags = { + project = var.project_tag + } +} + +resource "aws_vpc" "cloud_vpc" { + # we only will create this vpc if vpc_id is not passed in as a variable + count = var.vpc_id != null ? 0 : 1 + # This is a large vpc, 256 x 256 IPs available + cidr_block = "10.0.0.0/16" + enable_dns_support = true + enable_dns_hostnames = true + tags = { + Name = "${var.name_tag} VPC" + Project = var.project_tag + } +} + + +data "aws_vpc" "pre-provisioned" { + # the pre-provisioned VPC will be returned if vpc_id matches an existing VPC + count = var.vpc_id != null ? 1 : 0 + id = var.vpc_id +} + +resource "aws_subnet" "main" { + count = var.subnet_id != null ? 0 : 1 + vpc_id = local.vpc.id + + # If a subnet_cidr variable is passed explicitly, we use that, + # otherwise, divide the VPC by four and use 1/4 for a new subnet + cidr_block = var.subnet_cidr != null ? var.subnet_cidr : cidrsubnet(one(data.aws_vpc.pre-provisioned[*]).cidr_block, 2, var.subnet_quartile - 1) + + map_public_ip_on_launch = true + + availability_zone = var.availability_zone + + tags = { + Name = "${var.name_tag} Subnet" + Project = var.project_tag + } +} + + +data "aws_subnet" "pre-provisioned" { + # the pre-provisioned Subnet will be returned if subnet_id matches an existing Subnet + count = var.subnet_id != null ? 1 : 0 + id = var.subnet_id +} + + +# here we assign local variables for both the VPC and the Subnet we'll need to refer to to deploy further resources below: +# use of the one() function is needed to ensure only a single value is assigned, rather than a tuple/set +locals { + vpc = var.vpc_id != null ? one(data.aws_vpc.pre-provisioned[*]) : one(aws_vpc.cloud_vpc[*]) + subnet = var.subnet_id != null ? one(data.aws_subnet.pre-provisioned[*]) : one(aws_subnet.main[*]) + +} + + +resource "aws_internet_gateway" "gw" { + vpc_id = local.vpc.id + tags = { + Name = "${var.name_tag} Internet Gateway" + Project = var.project_tag + } +} + +resource "aws_route_table" "default" { + vpc_id = local.vpc.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = one(aws_internet_gateway.gw[*].id) + } + tags = { + Name = "${var.name_tag} Route Table" + Project = var.project_tag + } +} + +resource "aws_route_table_association" "main" { + subnet_id = one(aws_subnet.main[*].id) + route_table_id = one(aws_route_table.default[*].id) +} + + +resource "aws_efs_file_system" "main_efs" { + encrypted = false + availability_zone_name = var.availability_zone + tags = { + Name = "${var.name_tag} EFS" + Project = var.project_tag + } +} + + +resource "aws_efs_mount_target" "mount_target_main_efs" { + subnet_id = local.subnet.id + security_groups = [aws_security_group.efs_sg.id] + file_system_id = aws_efs_file_system.main_efs.id +} + + +### AMI Options +### Specify aws_ami.*****.id in aws_instance section below + +######################## +# CentOS 7 CentOS AMI +# ami-033adaf0b583374d4 +######################## + +data "aws_ami" "centos_7" { + owners = ["125523088429"] # CentOS.org + most_recent = true + + filter { + name = "name" + values = ["CentOS Linux 7 *"] + } + + filter { + name = "architecture" + values = ["x86_64"] + } + + filter { + name = "root-device-type" + values = ["ebs"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } +} + + +######################## +# CentOS 7 AMI by AWS - EOL +# 2023-04-17 us-east-2 id +# ami-05a36e1502605b4aa +######################## + +data "aws_ami" "centos_7_aws" { + owners = ["679593333241"] # AWS Marketplace + most_recent = true + + filter { + name = "description" + values = ["CentOS-7*"] + } + + filter { + name = "architecture" + values = ["x86_64"] + } + + filter { + name = "root-device-type" + values = ["ebs"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } +} + +################################### +# RHEL 8 +# 2023-10-06: ami-057094267c651958e +# AMI name: RHEL-8.7.0_HVM-20230330-x86_64-56-Hourly2-GP2 +# Owner account ID 309956199498 Red Hat +################################## + +data "aws_ami" "rhel_8" { + owners = ["309956199498"] + most_recent = true + + filter { + name = "name" + values = ["RHEL-8.7.0_HVM-20230330-x86_64-56-Hourly2-GP2"] + } + + filter { + name = "architecture" + values = ["x86_64"] + } + + filter { + name = "root-device-type" + values = ["ebs"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } +} + + +################################### +# Centos Stream 9 - untested +# ami-0c2abda83f1b9e09d +# AMI name: CentOS Stream 9 x86_64 +# Owner account ID 125523088429 +################################## + +data "aws_ami" "centos_stream_9" { + owners = ["125523088429"] # CentOS Official CPE + most_recent = true + + filter { + name = "description" + values = ["CentOS Stream 9 *"] + } + + filter { + name = "architecture" + values = ["x86_64"] + } + + filter { + name = "root-device-type" + values = ["ebs"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } +} + + +# Work around to get a public IP assigned when using EFA +resource "aws_eip" "head_node" { + depends_on = [aws_internet_gateway.gw] + vpc = true + instance = aws_instance.head_node.id + tags = { + Name = "${var.name_tag} Elastic IP" + Project = var.project_tag + } +} + + +# efa enabled node +resource "aws_instance" "head_node" { + + # Base CentOS 7 AMI, can use either AWS's marketplace, or direct from CentOS + # Choosing direct from CentOS as it is more recent + + ################################# + ### Specify which AMI to use here + ### Only CentOS 7 has been thoroughly tested + ############################################# + + ami = data.aws_ami.rhel_8.id + # ami = data.aws_ami.centos_stream_8.id + # ami = data.aws_ami.centos_7.id + # ami = data.aws_ami.centos_7_aws.id + + metadata_options { + http_endpoint = "enabled" + http_tokens = "required" + } + + instance_type = var.instance_type + cpu_threads_per_core = 2 + + root_block_device { + encrypted = true + delete_on_termination = true + volume_size = 24 + volume_type = "gp3" + tags = { + Name = "${var.name_tag} Head Node" + Project = var.project_tag + } + + } + + depends_on = [aws_internet_gateway.gw, + aws_efs_file_system.main_efs, + aws_efs_mount_target.mount_target_main_efs] + + key_name = var.key_name + iam_instance_profile = aws_iam_instance_profile.cloud_sandbox_iam_instance_profile.name + + # user_data = data.template_file.init_instance.rendered + + user_data = templatefile("init_template.tpl", { efs_name = aws_efs_file_system.main_efs.dns_name, ami_name = "${var.name_tag}-${random_pet.ami_id.id}", aws_region = var.preferred_region, project = var.project_tag }) + + # associate_public_ip_address = true + network_interface { + network_interface_id = aws_network_interface.head_node.id + device_index = 0 + } + + # This logic isn't perfect since some ena instance types can be in a placement group also + placement_group = var.use_efa == true ? aws_placement_group.cloud_sandbox_placement_group.id : null + + tags = { + Name = "${var.name_tag} EC2 Head Node" + Project = var.project_tag + } +} + +# A random id to use when creating the AMI +# This needs a new id if the instance_id changes - otherwise it won't create a new AMI +resource "random_pet" "ami_id" { + keepers = { + #instance_id = aws_instance.head_node.id + ami_id = var.ami_id + } + length = 2 +} + +#data "template_file" "init_instance" { +# template = file("./init_template.tpl") +# vars = { +# efs_name = aws_efs_file_system.main_efs.dns_name +# ami_name = "${var.name_tag}-${random_pet.ami_id.id}" +# aws_region = var.preferred_region +# project = var.project_tag +# } + +#depends_on = [aws_efs_file_system.main_efs, +# aws_efs_mount_target.mount_target_main_efs] +#} + +# Can only attach efa adaptor to a stopped instance! +resource "aws_network_interface" "head_node" { + + subnet_id = local.subnet.id + description = "The network adaptor to attach to the head_node instance" + security_groups = [aws_security_group.base_sg.id, + aws_security_group.ssh_ingress.id, + aws_security_group.efs_sg.id] + + interface_type = var.use_efa == true ? "efa" : null + + tags = { + Name = "${var.name_tag} Head Node Network Adapter" + Project = var.project_tag + } +} diff --git a/terraform/security.tf b/terraform/security.tf index 30a9fb9..a638e94 100644 --- a/terraform/security.tf +++ b/terraform/security.tf @@ -48,7 +48,7 @@ resource "aws_security_group" "ssh_ingress" { from_port = 22 to_port = 22 protocol = "tcp" - cidr_blocks = [var.allowed_ssh_cidr] + cidr_blocks = var.allowed_ssh_cidr_list } tags = { Name = "${var.name_tag} SSH SG" diff --git a/terraform/variables.tf b/terraform/variables.tf index 8e6e74b..454f6bd 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -1,7 +1,7 @@ variable "preferred_region" { description = "Preferred region in which to launch EC2 instances. Defaults to us-east-1" type = string - default = "us-east-1" + default = "us-east-2" } variable "nameprefix" { @@ -13,7 +13,7 @@ variable "nameprefix" { variable "name_tag" { description = "Value of the Name tag for the EC2 instance" type = string - default = "IOOS-Cloud-Sandbox-Terraform" + default = "IOOS-Cloud-Sandbox" } variable "project_tag" { @@ -25,20 +25,20 @@ variable "project_tag" { variable "availability_zone" { description = "Availability zone to use" type = string - default = "us-east-1a" + default = "us-east-2b" } variable "instance_type" { description = "EC2 Instance Type" type = string - #default = "c5n.18xlarge" - default = "t3.medium" + default = "t3.medium" + } variable "use_efa" { description = "Attach EFA Network" type = bool - default = "true" + default = "false" } variable "key_name" { @@ -47,18 +47,24 @@ variable "key_name" { nullable = false } -variable "allowed_ssh_cidr" { +#variable "allowed_ssh_cidr" { +# description = "Public IP address/range allowed for SSH access" +# type = string +# nullable = false +#} + +variable "allowed_ssh_cidr_list" { description = "Public IP address/range allowed for SSH access" - type = string + type = list nullable = false } variable "public_key" { description = "Contents of the SSH public key to be used for authentication" type = string + sensitive = true nullable = false - validation { condition = length(var.public_key) > 8 && substr(var.public_key, 0, 8) == "ssh-rsa " error_message = "The public_key value must start with \"ssh-rsa \"." @@ -112,8 +118,8 @@ variable "subnet_quartile" { variable "managed_policies" { description = "The attached IAM policies granting machine permissions" default = ["arn:aws:iam::aws:policy/AmazonEC2FullAccess", - "arn:aws:iam::aws:policy/AmazonS3FullAccess", - "arn:aws:iam::aws:policy/AmazonFSxFullAccess"] + "arn:aws:iam::aws:policy/AmazonS3FullAccess", + "arn:aws:iam::aws:policy/AmazonFSxFullAccess"] } variable "ami_id" {