[ptxdist] [PATCH 1/1] ptxd_make_get: Add support for downloading s3:// urls via aws s3 cp

[ptxdist] [PATCH 1/1] ptxd_make_get: Add support for downloading s3:// urls via aws s3 cp

[jon]

From: Jon Ringle <jringle@gridpoint.com>

This allows downloading files stored in an AWS S3 bucket via an s3:// url
This allows you to point PTXCONF_SETUP_PTXMIRROR to an s3:// bucket

Signed-off-by: Jon Ringle <jringle@gridpoint.com>
---
 scripts/lib/ptxd_make_get.sh | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/scripts/lib/ptxd_make_get.sh b/scripts/lib/ptxd_make_get.sh
index ee6b01346..d575ac13e 100644
--- a/scripts/lib/ptxd_make_get.sh
+++ b/scripts/lib/ptxd_make_get.sh
@@ -270,6 +270,35 @@ ptxd_make_get_svn() {
 }
 export -f ptxd_make_get_svn
 
+#
+# in env:
+#
+# ${path}	: local file name
+# ${url}	: the url to download
+# ${opts[]}	: an array of options
+#
+ptxd_make_get_s3() {
+	local temp_file
+	set -- "${opts[@]}"
+	unset opts
+
+	local file="${url##*/}"
+
+	# remove any pending or half downloaded files
+	rm -f -- "${path}."*
+
+	temp_file="$(mktemp "${path}.XXXXXXXXXX")" || ptxd_bailout "failed to create tempfile"
+
+	aws s3 cp "${url}" "${temp_file}" && {
+		chmod 644 -- "${temp_file}" &&
+		touch -- "${temp_file}" &&
+		mv -- "${temp_file}" "${path}"
+	}
+
+	ptxd_make_serialize_put
+}
+export -f ptxd_make_get_s3
+
 
 #
 # check if download is disabled
@@ -412,6 +441,10 @@ ptxd_make_get() {
 			ptxd_make_get_download_permitted &&
 			ptxd_make_get_http && return
 			;;
+		s3://*)
+			ptxd_make_get_download_permitted &&
+			ptxd_make_get_s3 && return
+			;;
 		file*)
 			local thing="${url/file:\/\///}"
 
-- 
2.25.1

Re: [ptxdist] [PATCH 1/1] ptxd_make_get: Add support for downloading s3:// urls via aws s3 cp

[Michael Olbrich]

On Fri, Aug 19, 2022 at 06:39:13AM -0400, jon@ringle.org wrote:
> From: Jon Ringle <jringle@gridpoint.com>
> 
> This allows downloading files stored in an AWS S3 bucket via an s3:// url
> This allows you to point PTXCONF_SETUP_PTXMIRROR to an s3:// bucket
> 
> Signed-off-by: Jon Ringle <jringle@gridpoint.com>
> ---
>  scripts/lib/ptxd_make_get.sh | 33 +++++++++++++++++++++++++++++++++
>  1 file changed, 33 insertions(+)
> 
> diff --git a/scripts/lib/ptxd_make_get.sh b/scripts/lib/ptxd_make_get.sh
> index ee6b01346..d575ac13e 100644
> --- a/scripts/lib/ptxd_make_get.sh
> +++ b/scripts/lib/ptxd_make_get.sh
> @@ -270,6 +270,35 @@ ptxd_make_get_svn() {
>  }
>  export -f ptxd_make_get_svn
>  
> +#
> +# in env:
> +#
> +# ${path}	: local file name
> +# ${url}	: the url to download
> +# ${opts[]}	: an array of options
> +#
> +ptxd_make_get_s3() {
> +	local temp_file
> +	set -- "${opts[@]}"
> +	unset opts
> +
> +	local file="${url##*/}"

unused?

> +
> +	# remove any pending or half downloaded files
> +	rm -f -- "${path}."*
> +
> +	temp_file="$(mktemp "${path}.XXXXXXXXXX")" || ptxd_bailout "failed to create tempfile"
> +
> +	aws s3 cp "${url}" "${temp_file}" && {

	aws s3 ... || return
	...

	to avoid the { } and indention.

Michael

> +		chmod 644 -- "${temp_file}" &&
> +		touch -- "${temp_file}" &&
> +		mv -- "${temp_file}" "${path}"
> +	}
> +
> +	ptxd_make_serialize_put
> +}
> +export -f ptxd_make_get_s3
> +
>  
>  #
>  # check if download is disabled
> @@ -412,6 +441,10 @@ ptxd_make_get() {
>  			ptxd_make_get_download_permitted &&
>  			ptxd_make_get_http && return
>  			;;
> +		s3://*)
> +			ptxd_make_get_download_permitted &&
> +			ptxd_make_get_s3 && return
> +			;;
>  		file*)
>  			local thing="${url/file:\/\///}"
>  
> -- 
> 2.25.1
> 
> 
> 

-- 
Pengutronix e.K.                           |                             |
Steuerwalder Str. 21                       | http://www.pengutronix.de/  |
31137 Hildesheim, Germany                  | Phone: +49-5121-206917-0    |
Amtsgericht Hildesheim, HRA 2686           | Fax:   +49-5121-206917-5555 |