Skip to content
Commits on Source (7)
# [1.0.0-alpha.15](https://git.gitlab.arm.com/bazel/rules_curl/compare/v1.0.0-alpha.14...v1.0.0-alpha.15) (2025-01-27)
### Features
- allow make variable substitution of the URL ([729b41d](https://git.gitlab.arm.com/bazel/rules_curl/commit/729b41d7a3d51cb96c25efd3557bff7ecc10bf3e))
- remove curl_upload_manifest ([a9bc078](https://git.gitlab.arm.com/bazel/rules_curl/commit/a9bc07824e1dd2f659395b65ac206f59c627e2cd))
### BREAKING CHANGES
- removes `dst` attribute
- curl_upload_manifest removed
# [1.0.0-alpha.14](https://git.gitlab.arm.com/bazel/rules_curl/compare/v1.0.0-alpha.13...v1.0.0-alpha.14) (2025-01-13)
### Bug Fixes
......
* @bazel
* @Luke.Hackwell
[Documentation] @matthew.clarkson
[Documentation] @Luke.Hackwell
*.md
[Licensing] @matthew.clarkson
[Licensing] @Luke.Hackwell
/.reuse/dep5
/LICENSES/*
[Configuration] @matthew.clarkson
[Configuration] @Luke.Hackwell
.editorconfig
[CI] @matthew.clarkson
[CI] @Luke.Hackwell
.gitlab-ci.yml
[Release] @matthew.clarkson
[Release] @Luke.Hackwell
/.releaserc.yaml
[Node] @matthew.clarkson
[Node] @Luke.Hackwell
/package.json
/.npmrc
[Branding] @matthew.clarkson
[Branding] @Luke.Hackwell
/icon.svg
[Bazel] @matthew.clarkson
[Bazel] @Luke.Hackwell
/.bazelrc
/.bazelrc.ci
/.bazelignore
......
module(
name = "rules_curl",
version = "1.0.0-alpha.14",
version = "1.0.0-alpha.15",
bazel_compatibility = [
">=7.4.0",
],
......@@ -11,6 +11,17 @@ bazel_dep(name = "bazel_skylib", version = "1.5.0")
bazel_dep(name = "toolchain_utils", version = "1.0.0-beta.18")
bazel_dep(name = "ape", version = "1.0.0-beta.17")
bazel_dep(name = "rules_go", version = "0.48.1")
bazel_dep(name = "gazelle", version = "0.31.0")
go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps")
go_deps.from_file(go_mod = "//curl/upload:go.mod")
use_repo(
go_deps,
"com_github_google_shlex",
)
go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk")
go_sdk.download(version = "1.23.4")
export = use_extension("@toolchain_utils//toolchain/export:defs.bzl", "toolchain_export")
use_repo(export, "ape-curl")
......
......@@ -28,6 +28,9 @@
"https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8",
"https://bcr.bazel.build/modules/download_utils/1.0.0-beta.2/MODULE.bazel": "bced1551849a5d1ca00b985c0d267ab690af840f04c685f2c62f40e92f66fac0",
"https://bcr.bazel.build/modules/download_utils/1.0.0-beta.2/source.json": "0ab7ebbc57f39a7fe96190e01fe9773482bc4e3d465e9cd9b239bb44ad57791d",
"https://bcr.bazel.build/modules/gazelle/0.27.0/MODULE.bazel": "3446abd608295de6d90b4a8a118ed64a9ce11dcb3dda2dc3290a22056bd20996",
"https://bcr.bazel.build/modules/gazelle/0.30.0/MODULE.bazel": "f888a1effe338491f35f0e0e85003b47bb9d8295ccba73c37e07702d8d31c65b",
"https://bcr.bazel.build/modules/gazelle/0.31.0/MODULE.bazel": "0319690246f72d0b5d596724a0ea0da2fd823905643a042c95bc2c420438ddae",
"https://bcr.bazel.build/modules/gazelle/0.32.0/MODULE.bazel": "b499f58a5d0d3537f3cf5b76d8ada18242f64ec474d8391247438bf04f58c7b8",
"https://bcr.bazel.build/modules/gazelle/0.33.0/MODULE.bazel": "a13a0f279b462b784fb8dd52a4074526c4a2afe70e114c7d09066097a46b3350",
"https://bcr.bazel.build/modules/gazelle/0.34.0/MODULE.bazel": "abdd8ce4d70978933209db92e436deb3a8b737859e9354fb5fd11fb5c2004c8a",
......@@ -52,6 +55,9 @@
"https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e",
"https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5",
"https://bcr.bazel.build/modules/rules_cc/0.0.9/source.json": "1f1ba6fea244b616de4a554a0f4983c91a9301640c8fe0dd1d410254115c8430",
"https://bcr.bazel.build/modules/rules_go/0.33.0/MODULE.bazel": "a2b11b64cd24bf94f57454f53288a5dacfe6cb86453eee7761b7637728c1910c",
"https://bcr.bazel.build/modules/rules_go/0.38.1/MODULE.bazel": "fb8e73dd3b6fc4ff9d260ceacd830114891d49904f5bda1c16bc147bcc254f71",
"https://bcr.bazel.build/modules/rules_go/0.39.1/MODULE.bazel": "d34fb2a249403a5f4339c754f1e63dc9e5ad70b47c5e97faee1441fc6636cd61",
"https://bcr.bazel.build/modules/rules_go/0.41.0/MODULE.bazel": "55861d8e8bb0e62cbd2896f60ff303f62ffcb0eddb74ecb0e5c0cbe36fc292c8",
"https://bcr.bazel.build/modules/rules_go/0.42.0/MODULE.bazel": "8cfa875b9aa8c6fce2b2e5925e73c1388173ea3c32a0db4d2b4804b453c14270",
"https://bcr.bazel.build/modules/rules_go/0.46.0/MODULE.bazel": "3477df8bdcc49e698b9d25f734c4f3a9f5931ff34ee48a2c662be168f5f2d3fd",
......
......@@ -10,25 +10,6 @@ Add the following to `MODULE.bazel`:
bazel_dep(module_name = "rules_curl", version = "0.0.0")
```
## Providers
### ManifestInfo
Encapsulates the file to upload and its URL template (see [ManifestInfo][manifest_info])
Members:
- **file**: File
- **url**: string
### ManifestsInfo
The dependency set of `ManifestInfo`s
Members:
- **manifests**: depset of `ManifestInfo`s
## Usage
### curl_upload_file
......@@ -44,22 +25,3 @@ curl_upload_file(
)
```
### curl_upload_manifests
```py
load("@rules_curl//curl/upload/manifests:defs.bzl", "curl_upload_manifests")
curl_upload_manifests(
name = "upload",
srcs = [
":fixture_1",
":fixture_2",
],
url = "https://test.case",
)
```
> Note: `fixture_1` and `fixture_2` have to provide either `ManifestInfo` or `ManifestsInfo`
[manifest_info]: curl/upload/ManifestInfo.bzl
load("@rules_go//go:def.bzl", "go_binary")
go_binary(
name = "template",
srcs = ["template.go"],
pure = "on",
visibility = ["//curl/upload:__subpackages__"],
)
package main
import (
"bytes"
"flag"
"fmt"
"html/template"
"log"
"net/url"
"path/filepath"
"strconv"
"strings"
)
var manifest ManifestInfo = ManifestInfo{
URL: URLInfo{},
File: FileInfo{},
}
type StringInfo string
func (s StringInfo) String() string {
return string(s)
}
func (s *StringInfo) Set(value string) error {
buf := new(bytes.Buffer)
tmpl, err := template.New("templ").Parse(value)
if err != nil {
log.Fatal(err)
}
err = tmpl.Execute(buf, manifest)
if err != nil {
log.Fatal(err)
}
*s = StringInfo(buf.String())
return nil
}
type FileInfo struct {
Dirname StringInfo
Stem StringInfo
Extension StringInfo
}
func (f FileInfo) Basename() string {
extension := f.Extension.String()
stem := f.Stem.String()
if extension == "" {
return stem
}
return stem + "." + extension
}
func (f FileInfo) Path() string {
return f.Dirname.String() + "/" + f.Basename()
}
func (f FileInfo) String() string {
return f.Path()
}
func (f *FileInfo) Set(value string) error {
clean := filepath.Clean(value)
dirname := filepath.Dir(clean)
f.Dirname = StringInfo(dirname)
extension := filepath.Ext(value)
nodot := strings.TrimPrefix(extension, ".")
f.Extension = StringInfo(nodot)
base := filepath.Base(value)
stem := strings.TrimSuffix(base, extension)
f.Stem = StringInfo(stem)
return nil
}
type URLInfo struct {
Username *StringInfo
Password *StringInfo
Host StringInfo
Pathname StringInfo
Protocol StringInfo
}
func (u URLInfo) split() (string, *int) {
host := u.Host.String()
index := strings.LastIndexByte(host, ':')
if index == -1 {
return host, nil
}
hostname := host[:index]
port, err := strconv.Atoi(host[index+1:])
if err != nil {
log.Fatal(err)
}
return hostname, &port
}
func (u URLInfo) Hostname() string {
hostname, _ := u.split()
return hostname
}
func (u URLInfo) Port() *int {
_, port := u.split()
return port
}
func (u URLInfo) Origin() string {
return u.Protocol.String() + "//" + u.Host.String()
}
func (u URLInfo) Auth() (auth string) {
if u.Username != nil {
auth += string(*u.Username)
}
if u.Password != nil {
auth += ":" + string(*u.Password)
}
return
}
func (u URLInfo) Href() (href string) {
href += u.Protocol.String() + "//"
if auth := u.Auth(); auth != "" {
href += auth + "@"
}
href += u.Host.String()
href += u.Pathname.String()
return
}
func (u URLInfo) String() string {
return u.Href()
}
func (u *URLInfo) Set(value string) error {
parsed, err := url.Parse(value)
if err != nil {
return err
}
if username := parsed.User.Username(); username != "" {
info := StringInfo(username)
u.Username = &info
}
if password, set := parsed.User.Password(); set {
info := StringInfo(password)
u.Password = &info
}
u.Host = StringInfo(parsed.Host)
u.Pathname = StringInfo(parsed.Path)
u.Protocol = StringInfo(parsed.Scheme + ":")
return err
}
type ManifestInfo struct {
URL URLInfo
File FileInfo
}
func main() {
flag.Var(&manifest.File, "file", "The file path to use for templating")
flag.Var(&manifest.File.Dirname, "dirname", "The directory for the destination file")
flag.Var(&manifest.File.Stem, "stem", "The basename of the destination file")
flag.Var(&manifest.File.Extension, "extension", "The extension of the destination file")
flag.Var(&manifest.URL, "url", "URL to use for templating")
flag.Var(&manifest.URL.Host, "host", "The domain name of the URL")
flag.Var(&manifest.URL.Protocol, "scheme", "The scheme for the URL")
flag.Func("pathname", "A location in a hierachical structure of the URL", func(s string) error {
return manifest.URL.Pathname.Set(s)
})
flag.Func("origin", "The origin of the represented URL.", func(s string) error {
index := strings.Index(s, "//")
if index == -1 {
manifest.URL.Protocol = StringInfo(s)
return nil
}
manifest.URL.Host = StringInfo(s[index+2:])
manifest.URL.Protocol = StringInfo(s[:index])
return nil
})
t := flag.String("template", "{{.URL.Href}}/{{.File.Path}}", "The Go template to render")
flag.Parse()
tmpl, err := template.New("url").Parse(*t)
if err != nil {
log.Fatal(err)
}
buf := new(bytes.Buffer)
err = tmpl.Execute(buf, manifest)
if err != nil {
log.Fatal(err)
}
_, err = url.Parse(buf.String())
if err != nil {
log.Fatal(err)
}
fmt.Printf("%s", buf.String())
}
exports_files([
"posix.tmpl.sh",
"nt.tmpl.bat",
])
load("@rules_go//go:def.bzl", "go_binary")
alias(
name = "script",
actual = select(
{
"@toolchain_utils//toolchain/constraint/os:windows": ":nt.tmpl.bat",
"//conditions:default": ":posix.tmpl.sh",
},
no_match_error = "No script template available for `curl_upload_manifests`",
),
visibility = ["//curl/upload:__subpackages__"],
)
sh_binary(
name = "csv",
srcs = ["csv.sh"],
go_binary(
name = "upload",
srcs = ["upload.go"],
visibility = ["//curl/upload:__subpackages__"],
deps = [
"@rules_go//go/runfiles:go_default_library",
"@com_github_google_shlex//:shlex"
],
pure = "on"
)
exports_files([
"go.mod",
])
load("@bazel_skylib//lib:types.bzl", "types")
visibility("//...")
def init(file, url):
"""
Initializes a `CurlUploadManifestInfo` provider.
To be used with `curl_upload_manifests`
Args:
file: The file to upload.
url: A Starlark string template for the URL, which can include:
- {{.URL.Href}}: the provided URL
- {{.URL.Auth}}: the user:password auth
- {{.URL.Origin}}: the origin of the represented URL
- {{.URL.Port}}: the port number of the URL
- {{.URL.Host}}: the hostname followed by the port if not empty
- {{.URL.Hostname}}: the domain name of the URL
- {{.URL.Protocol}}: the protocol scheme of the URL
- {{.URL.Pathname}}: a location in a hierarchical structure
- {{.File.Path}}: the full path of the file
- {{.File.Basename}}: the basename of the file
- {{.File.Dirname}}: the directory of the file
- {{.File.Stem}}: the file name without extension
- {{.File.Extension}}: the extension of the file
Returns:
A mapping of keywords for the `curl_upload_manifest_info` raw constructor.
"""
if type(file) != "File":
fail("`CurlUploadManifestInfo.file` must be a `file`: {}".format(file))
if file.is_directory:
fail("`CurlUploadManifestInfo.file` must not be a directory: {}".format(file))
if not types.is_string(url):
fail("`CurlUploadManifestInfo.url` must be a `str`: {}".format(url))
if url.find(",") != -1:
fail("`CurlUploadManifestInfo.url` must not have comma `,` sign: {}".format(url))
return {
"file": file,
"url": url,
}
CurlUploadManifestInfo, curl_upload_manifest_info = provider(
"A file to upload with cURL.",
fields = ["file", "url"],
init = init,
)
# Provide some convenience imports
ManifestInfo = CurlUploadManifestInfo
manifest_info = curl_upload_manifest_info
Info = CurlUploadManifestInfo
info = curl_upload_manifest_info
load("@bazel_skylib//lib:types.bzl", "types")
visibility("//...")
def init(manifests):
"""
Initializes a `CurlUploadManifestsInfo` provider.
To be used with `curl_upload_manifests`
Args:
manifests: The dependency set of `CurlUploadManifestInfo`s
Returns:
A mapping of keywords for the `curl_upload_manifests_info` raw constructor.
"""
if not types.is_depset(manifests):
fail("`CurlUploadManifestsInfo.url` must be a `depset`: {}".format(manifests))
return {
"manifests": manifests,
}
CurlUploadManifestsInfo, curl_upload_manifests_info = provider(
"Files to upload with cURL.",
fields = ["manifests"],
init = init,
)
# Provide some convenience imports
ManifestsInfo = CurlUploadManifestsInfo
manifests_info = curl_upload_manifests_info
Info = CurlUploadManifestsInfo
info = curl_upload_manifests_info
#! /usr/bin/env sh
set -o errexit -o nounset
OUT="${1}"
shift
readonly OUT
for ARG in "${@}"; do
if test -n "${ARG}"; then
printf >>"${OUT}" '%s\n' "${ARG}"
fi
done
load("@rules_curl//curl/upload:ManifestInfo.bzl", "ManifestInfo", "manifest_info")
visibility("//curl/...")
DOC = """Upload a file to a URL endpoint with cURL.
......@@ -8,7 +6,6 @@ DOC = """Upload a file to a URL endpoint with cURL.
file(
name = "upload_file",
src = ":data",
dst = "data"
url = "https://host.name.to.upload",
)
```
......@@ -20,12 +17,8 @@ ATTRS = {
mandatory = True,
allow_single_file = True,
),
"dst": attr.string(
doc = "The filename to upload as.",
mandatory = True,
),
"url": attr.string(
doc = "URL endpoint for file upload.",
doc = "URL endpoint for file upload. Subject to 'Make variable' expansion.",
mandatory = True,
),
"retry": attr.int(
......@@ -36,66 +29,52 @@ ATTRS = {
doc = "The seconds to wait before attempting a upload retry.",
default = 1,
),
"_script": attr.label(
doc = "The template that is expanded into the upload binary.",
default = "//curl/upload:script",
allow_single_file = True,
),
"_template": attr.label(
default = "//curl/template:template",
cfg = "exec",
"_upload": attr.label(
default = "//curl/upload",
allow_single_file = True,
executable = True,
),
"_csv": attr.label(
doc = "CSV tool",
default = "//curl/upload:csv",
cfg = "exec",
executable = True,
),
),
}
def _runfile(label, file):
path = file.short_path
if path.startswith("../"):
return path.removeprefix("../")
return "{}/{}".format(label.workspace_name or "_main", path)
def implementation(ctx):
curl = ctx.toolchains["//curl/toolchain/curl:type"]
csv = ctx.actions.declare_file("{}.upload.csv".format(ctx.label.name))
href = ctx.attr.url.rstrip("/").replace(",", "%2C")
args = ctx.actions.args()
# SRC, DST, TEMPLATE, URL
args.add("{},{},{},{}".format(ctx.file.src.short_path, ctx.attr.dst, "{{.URL.Href}}/{{.File.Path}}", href))
ctx.actions.run(
outputs = [csv],
inputs = [ctx.file.src],
arguments = [csv.path, args],
executable = ctx.executable._csv,
mnemonic = "PrepareUploadCSV",
)
if ctx.file.src.is_directory:
fail("'src' must be a file not a directory.")
executable = ctx.actions.declare_file("{}.sh".format(ctx.label.name))
# Do the 'Make variable' expansion on the URL.
url = ctx.attr.url
for k, v in ctx.var.items():
url = url.replace("$({})".format(k), v)
substitutions = ctx.actions.template_dict()
substitutions.add("{{curl}}", str(curl.executable.short_path))
substitutions.add("{{retry}}", str(ctx.attr.retry))
substitutions.add("{{retry_delay}}", str(ctx.attr.retry_delay))
substitutions.add("{{csv}}", str(csv.short_path))
substitutions.add("{{template}}", str(ctx.executable._template.short_path))
substitutions.add("{{directory}}", str(ctx.file.src.short_path))
arguments = ctx.actions.declare_file("{}.args".format(ctx.label.name))
args = ctx.actions.args()
args.add("src", _runfile(ctx.file.src.owner, ctx.file.src))
args.add("url", url)
args.add("curl", _runfile(curl.executable.owner, curl.executable))
args.add("retry", str(ctx.attr.retry))
args.add("retry-delay", str(ctx.attr.retry_delay))
ctx.actions.write(output = arguments, content = args)
ctx.actions.expand_template(
template = ctx.file._script,
executable = ctx.actions.declare_file(ctx.label.name)
ctx.actions.symlink(
output = executable,
computed_substitutions = substitutions,
target_file = ctx.executable._upload,
is_executable = True,
)
files = depset([executable])
runfiles = ctx.runfiles([curl.executable, ctx.file.src, csv])
runfiles = runfiles.merge(ctx.attr.src.default_runfiles)
files = depset([executable, arguments])
root_symlinks = {"upload.args": arguments}
runfiles = ctx.runfiles([ctx.executable._upload, curl.executable, ctx.file.src], root_symlinks = root_symlinks)
runfiles = runfiles.merge(curl.default.default_runfiles)
runfiles = runfiles.merge(ctx.attr._template.default_runfiles)
runfiles = runfiles.merge(ctx.attr._csv.default_runfiles)
return DefaultInfo(
executable = executable,
......
module curl_runner
go 1.19
require (
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
)
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
load("//curl/upload:ManifestInfo.bzl", _Manifest = "Info")
visibility("public")
CurlUploadManifestInfo = _Manifest
load(":rule.bzl", _manifests = "manifests")
load("//curl/upload:ManifestInfo.bzl", _Manifest = "Info")
load("//curl/upload:ManifestsInfo.bzl", _Manifests = "Info")
visibility("public")
curl_upload_manifests = _manifests
CurlUploadManifestInfo = _Manifest
CurlUploadManifestsInfo = _Manifests
load("//curl/upload:ManifestInfo.bzl", "ManifestInfo")
load("//curl/upload:ManifestsInfo.bzl", "ManifestsInfo")
visibility("//curl/...")
DOC = """Upload bunch of files to a URL endpoint with cURL.
The `srcs` must provide `ManifestInfo` or `ManifestsInfo`.
```py
file(
name = "upload_files",
srcs = [
":data",
],
url = "https://host.name.to.upload",
)
```
"""
ATTRS = {
"srcs": attr.label_list(
doc = "Files to be uploaded.",
mandatory = True,
providers = [
[ManifestInfo],
[ManifestsInfo],
],
allow_files = False,
),
"url": attr.string(
doc = "URL endpoint for files to upload.",
mandatory = True,
),
"retry": attr.int(
doc = "The number of retry attempts for every file.",
default = 3,
),
"retry_delay": attr.int(
doc = "The seconds to wait before attempting an upload retry.",
default = 1,
),
"_script": attr.label(
doc = "The template that is expanded into the upload binary.",
default = "//curl/upload:script",
allow_single_file = True,
),
"_template": attr.label(
default = "//curl/template:template",
cfg = "exec",
allow_single_file = True,
executable = True,
),
"_csv": attr.label(
doc = "CSV tool",
default = "//curl/upload:csv",
cfg = "exec",
executable = True,
),
}
def implementation(ctx):
curl = ctx.toolchains["//curl/toolchain/curl:type"]
csv = ctx.actions.declare_file("{}.upload.csv".format(ctx.label.name))
manifests = depset(
direct = [src[ManifestInfo] for src in ctx.attr.srcs if ManifestInfo in src],
transitive = [src[ManifestsInfo].manifests for src in ctx.attr.srcs if ManifestsInfo in src],
)
href = ctx.attr.url.rstrip("/").replace(",", "%2C")
def _to_string(m):
# SRC, DST, TEMPLATE, URL
return "{},{},{},{}".format(m.file.short_path, m.file.short_path, m.url, href)
args = ctx.actions.args()
args.add_all(manifests, map_each = _to_string, allow_closure = True)
ctx.actions.run(
outputs = [csv],
arguments = [csv.path, args],
executable = ctx.executable._csv,
mnemonic = "PrepareUploadCSV",
)
executable = ctx.actions.declare_file("{}.sh".format(ctx.label.name))
substitutions = ctx.actions.template_dict()
substitutions.add("{{curl}}", str(curl.executable.short_path))
substitutions.add("{{retry}}", str(ctx.attr.retry))
substitutions.add("{{retry_delay}}", str(ctx.attr.retry_delay))
substitutions.add("{{csv}}", str(csv.short_path))
substitutions.add("{{template}}", str(ctx.executable._template.short_path))
ctx.actions.expand_template(
template = ctx.file._script,
output = executable,
computed_substitutions = substitutions,
is_executable = True,
)
files = depset([executable])
runfiles = ctx.runfiles([curl.executable, csv, ctx.executable._template] + [m.file for m in manifests.to_list()])
runfiles = runfiles.merge(curl.default.default_runfiles)
runfiles = runfiles.merge(ctx.attr._template.default_runfiles)
return DefaultInfo(
executable = executable,
files = files,
runfiles = runfiles,
)
curl_upload_manifests = rule(
doc = DOC,
attrs = ATTRS,
implementation = implementation,
toolchains = ["//curl/toolchain/curl:type"],
executable = True,
)
manifests = curl_upload_manifests
@echo off
# TODO: implement Windows Batch for `curl_upload`
exit /b 121