From f06bd8a7d671a722e003c411888251203e011e40 Mon Sep 17 00:00:00 2001 From: helly25 Date: Fri, 25 Apr 2025 19:00:12 +0000 Subject: [PATCH 01/10] Add versions Add versions/distributions starting at 17.0.0. Add a test that verifies which verions we are looking for and which distributions we can reach. The test shows that a few distributions currently cannot be reached. That is, we have their distribution entries but cannot ever detect them. --- toolchain/internal/BUILD.bazel | 15 ++ toolchain/internal/llvm_distributions.bzl | 210 +++++++++++++++++- .../internal/llvm_distributions.golden.txt | 27 +++ utils/llvm_checksums.sh | 39 ++-- 4 files changed, 272 insertions(+), 19 deletions(-) create mode 100644 toolchain/internal/llvm_distributions.golden.txt diff --git a/toolchain/internal/BUILD.bazel b/toolchain/internal/BUILD.bazel index 35d620f84..ac917f723 100644 --- a/toolchain/internal/BUILD.bazel +++ b/toolchain/internal/BUILD.bazel @@ -12,4 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@bazel_skylib//rules:diff_test.bzl", "diff_test") +load("llvm_distributions.bzl", "write_distributions") + exports_files(["template.modulemap"]) + +write_distributions( + name = "llvm_distributions", + testonly = True, + visibility = ["//visibility:private"], +) + +diff_test( + name = "llvm_distributions_test", + file1 = "llvm_distributions.golden.txt", + file2 = "llvm_distributions", +) diff --git a/toolchain/internal/llvm_distributions.bzl b/toolchain/internal/llvm_distributions.bzl index 380db86cb..d8bb93312 100644 --- a/toolchain/internal/llvm_distributions.bzl +++ b/toolchain/internal/llvm_distributions.bzl @@ -405,6 +405,7 @@ _llvm_distributions = { "clang+llvm-17.0.1-aarch64-linux-gnu.tar.xz": "d2eaca72ce3aab0b343e01b2233303628ff43a43a6107dca1aa8d3039da847f5", "clang+llvm-17.0.1-amd64-pc-solaris2.11.tar.xz": "153b8b650705390cc3b4ff739b061f5bff87542531c002c039d52c00781559f7", "clang+llvm-17.0.1-arm64-apple-darwin22.0.tar.xz": "d5678bc475c42c3ab7c0ee35ebd95534d81b6816507cfc9e42d86136d8315ebc", + "clang+llvm-17.0.1-armv7a-linux-gnueabihf.tar.gz": "1ca3b7adeee14c656e0689c76cea962a644fb6dba9ecda894505506f837f9d69", "clang+llvm-17.0.1-final_powerpc64-ibm-aix-7.2.tar.xz": "f5d9bf5822a775d4b10e7af035076e1779983dee1b05b3f57af2674231bcf678", "clang+llvm-17.0.1-powerpc64le-linux-rhel-8.8.tar.xz": "937394cd44c5eb81aae8d1c66b6d3b930ebd2b013ac6493d17f33f5083281d37", "clang+llvm-17.0.1-sparc64-unknown-linux-gnu.tar.xz": "0ab5f6a9b19ec968628d987d9430a033801c78bee65fbf40c72da660ff401f4d", @@ -414,6 +415,7 @@ _llvm_distributions = { "clang+llvm-17.0.2-aarch64-linux-gnu.tar.xz": "b08480f2a77167556907869065b0e0e30f4d6cb64ecc625d523b61c22ff0200f", "clang+llvm-17.0.2-amd64-pc-solaris2.11.tar.xz": "8e98c6015202575407f5580bed9a9b58d3bdc3e5d64e39289189b491949b957f", "clang+llvm-17.0.2-arm64-apple-darwin22.0.tar.xz": "dfb3226b3e16f5b8d3882f3ff0e8ebf40b26dd1e97d879197430b930d773ea84", + "clang+llvm-17.0.2-armv7a-linux-gnueabihf.tar.gz": "2204da50c85db65def57b11bd0d96abdc96808cf410403daf7aa9c86f2b9e732", "clang+llvm-17.0.2-powerpc64-ibm-aix-7.2.tar.xz": "c0175b48bf72c621316f3fc7ec4662163d4e17718b179f967d75149d7cfeee80", "clang+llvm-17.0.2-powerpc64le-linux-rhel-8.8.tar.xz": "ef19116996a1966a4fa6e261c86eef7b807e5f39a963dc914b5547976336ab1b", "clang+llvm-17.0.2-sparc64-unknown-linux-gnu.tar.xz": "950d1ef440f17e29c4201450ad619d3b4a37a0bbf15f19ce03195e0b4da7d73f", @@ -423,12 +425,14 @@ _llvm_distributions = { # 17.0.3 "clang+llvm-17.0.3-aarch64-linux-gnu.tar.xz": "289da98e4cbc157153e987ff24ce835717a36cfab03ecd03bf359378ee4ae9d7", "clang+llvm-17.0.3-arm64-apple-darwin22.0.tar.xz": "da452a1aa33954c123d5264bd849ebc572a28e8511b868b43e82d6960fda60d7", + "clang+llvm-17.0.3-armv7a-linux-gnueabihf.tar.gz": "5da6f3a350a34f8401125d31aeef85bc2deda04601b9b703f62356e81516e73c", "clang+llvm-17.0.3-powerpc64-ibm-aix-7.2.tar.xz": "86b05883c17ddb4b7e9ad6a6a88e78311c117fdc03415fa47293e12e6e2810ff", "clang+llvm-17.0.3-powerpc64le-linux-rhel-8.8.tar.xz": "fcba4ac2a717762ff1b5fe482a811648837d7dc7bf7b654702c80f2fa044d07d", # 17.0.4 "clang+llvm-17.0.4-aarch64-linux-gnu.tar.xz": "18b326b3e17168fc423726b5059b4d55b6070d49408e51440ad3fca2ebb37779", "clang+llvm-17.0.4-arm64-apple-darwin22.0.tar.xz": "5d514fa64a290dca53288ce859e6ec59a0b48198b3a5b27ca53b6fe80a977b8d", + "clang+llvm-17.0.4-armv7a-linux-gnueabihf.tar.gz": "6da0b41a942bd5020966511722e8917260349628b6a77aab916ca2c244cecafd", "clang+llvm-17.0.4-powerpc64-ibm-aix-7.2.tar.xz": "54d4d6b91624597e0c0b15264c4c8e57092f521247b87ba6f1297db339ac6e2b", "clang+llvm-17.0.4-powerpc64le-linux-rhel-8.8.tar.xz": "2e3ac8b7288ed5d5c3549e457332bbf3c913022fdd7cfbe13fde46448f76d136", "clang+llvm-17.0.4-x86_64-linux-gnu-ubuntu-22.04.tar.xz": "6b45be6c0483b7ee3f63981678093b731fd9f4ea6987b4ceb6efde21890ffca7", @@ -436,6 +440,7 @@ _llvm_distributions = { # 17.0.5 "clang+llvm-17.0.5-aarch64-linux-gnu.tar.xz": "ee12126c404d42a0723ff3a4449470b5570fe5ce610be9d9baee88a6d27701d2", "clang+llvm-17.0.5-arm64-apple-darwin22.0.tar.xz": "6c9aa227800d30d39c28dadbd72c15442e0d9b6813efb2aaa66a478630b7f0c6", + "clang+llvm-17.0.5-armv7a-linux-gnueabihf.tar.gz": "b7978d073e250ed66d5e8a5136026460200db1951ce75d402976fde4c2f0c3d8", "clang+llvm-17.0.5-powerpc64-ibm-aix-7.2.tar.xz": "b5da095901fe604f562363cf9611d6ca73e13d81831a96518823d690babc608f", "clang+llvm-17.0.5-powerpc64le-linux-rhel-8.8.tar.xz": "11aace89d7881b694a05d1e93de3c78a31e141d0df1401491d67f73020bc3df2", "clang+llvm-17.0.5-x86_64-linux-gnu-ubuntu-22.04.tar.xz": "5a3cedecd8e2e8663e84bec2f8e5522b8ea097f4a8b32637386f27ac1ca01818", @@ -444,6 +449,7 @@ _llvm_distributions = { "clang+llvm-17.0.6-aarch64-linux-gnu.tar.xz": "6dd62762285326f223f40b8e4f2864b5c372de3f7de0731cb7cd55ca5287b75a", "clang+llvm-17.0.6-amd64-pc-solaris2.11.tar.xz": "8feb660750a4d24b18d8e894fbccf26bd0dfbc92581d202ec9057f00f3fbf232", "clang+llvm-17.0.6-arm64-apple-darwin22.0.tar.xz": "1264eb3c2a4a6d5e9354c3e5dc5cb6c6481e678f6456f36d2e0e566e9400fcad", + "clang+llvm-17.0.6-armv7a-linux-gnueabihf.tar.gz": "7a51ea063f74fb1b7c0389455916cec52c98d1b51da44d6ebc8232014d7af3d1", "clang+llvm-17.0.6-powerpc64-ibm-aix-7.2.tar.xz": "3aeda4bb5808db2e47bde60cc49b15b869114e3681092413f7b297345d2e13ce", "clang+llvm-17.0.6-powerpc64le-linux-rhel-8.8.tar.xz": "04e18072797920c2b5e9bdf0c3ee9e5a61adf76bd5ffeb438fafd9e32fc48b62", "clang+llvm-17.0.6-sparcv9-sun-solaris2.11.tar.xz": "b7df7b383679af98640640f88114f461f38a6efdfe7c369692b0675751ac2773", @@ -452,6 +458,7 @@ _llvm_distributions = { # 18.1.0 "clang+llvm-18.1.0-aarch64-linux-gnu.tar.xz": "32faaad5b6e072d763a603f7c51e4ee63e2d82c16e945524a539df84e3f2b058", "clang+llvm-18.1.0-amd64-pc-solaris2.11.tar.xz": "c352b81dd6add029e3def54a7b90387bb1df15f76497adac0b9f305694eb2d8c", + "clang+llvm-18.1.0-armv7a-linux-gnueabihf.tar.gz": "bfbd3bb71f4a1aaf8f1e13cf681e16a54f1029e7f5c85492812bf93a1d893dc8", "clang+llvm-18.1.0-powerpc64-ibm-aix-7.2.tar.xz": "cc9bcf2b2132c158a71f7f3971d105454131701c25767f97e977c568418aff89", "clang+llvm-18.1.0-powerpc64le-linux-rhel-8.8.tar.xz": "730c40a0c79d89ca8875c2004fd49180e9b65585b24f68728232b06b3d8bda32", "clang+llvm-18.1.0-sparcv9-sun-solaris2.11.tar.xz": "e871f472ceafbe0197cff81d7240552e45e55ead00fe82f4fb326af32bbfb657", @@ -460,6 +467,7 @@ _llvm_distributions = { # 18.1.1 "clang+llvm-18.1.1-aarch64-linux-gnu.tar.xz": "6815ef3c314566605f90cff7922ff3ef5a6eaaf854604e4add6a170e6e98389f", + "clang+llvm-18.1.1-armv7a-linux-gnueabihf.tar.gz": "c87928c9d9d9c4c0eedfed1fb49216b9c52c377ccdd0242f7145401b9aea51f7", "clang+llvm-18.1.1-powerpc64-ibm-aix-7.2.tar.xz": "c900418e781d0de1f316fcce50ffeca903fa15d97df0dd90f6ac4bd2b43105d4", "clang+llvm-18.1.1-powerpc64le-linux-rhel-8.8.tar.xz": "7415429a0c0eceeacedc00b3f99f9a869909682fab130c2e514f240379539741", "clang+llvm-18.1.1-x86_64-pc-windows-msvc.tar.xz": "79ea242c0fbd66c632ed3aaebf6f821c1e4c03140497c67ea750443eb36bfc5d", @@ -467,6 +475,7 @@ _llvm_distributions = { # 18.1.2 "clang+llvm-18.1.2-aarch64-linux-gnu.tar.xz": "aa9d6c6e70cbe2344be1f4b780525a9a4feb70a6e4fa46ea67822f0e7f839c21", "clang+llvm-18.1.2-amd64-pc-solaris2.11.tar.xz": "83ca7644b5eebf5ac55014e628d0bbe685a79416d70a0d80d24ece0ddfc05c6d", + "clang+llvm-18.1.2-armv7a-linux-gnueabihf.tar.gz": "5260702615952f25bc715c4aaa286c85d44c20a9a662357a7774841539560fe3", "clang+llvm-18.1.2-powerpc64-ibm-aix-7.2.tar.xz": "ad7351206905f61933be5937017fc454995d287346f7f0325c48c4552803af87", "clang+llvm-18.1.2-powerpc64le-linux-rhel-8.8.tar.xz": "0dc4831dab74f47691dab934a52a055ea8fae6bfeec2ed5261991146b38f1cf3", "clang+llvm-18.1.2-sparcv9-sun-solaris2.11.tar.xz": "b719027e8423296f06375ee151652623b0a1df46848dac0bb2614210e5bd233e", @@ -496,12 +505,14 @@ _llvm_distributions = { # 18.1.6 "clang+llvm-18.1.6-aarch64-linux-gnu.tar.xz": "bcb3d53d3bd1027bc7f26544dff8cdc5ff74776add6eb994047326b284147a90", "clang+llvm-18.1.6-amd64-pc-solaris2.11.tar.xz": "abdf9e930c0069b97cff69156c62e97056f8f9ec24d15cdea743ac555887436e", + "clang+llvm-18.1.6-armv7a-linux-gnueabihf.tar.gz": "30264de61eaed2f860217a71e701d2ce3d1821acea0e0239bc6a8457ff4586f2", "clang+llvm-18.1.6-powerpc64le-linux-rhel-8.8.tar.xz": "201c8d784acf9e3553a00078bd2e4007134957bd4541706fafe9c7c0583c3cd6", "clang+llvm-18.1.6-sparcv9-sun-solaris2.11.tar.xz": "da65c1abea553c17fd22ae3de51c70f4ff0789e95019fb4d95f05371b830e090", "clang+llvm-18.1.6-x86_64-pc-windows-msvc.tar.xz": "479e9e77b9d114721a7168718c894343ac01c397db3499e8a3002ee7a3903d54", # 18.1.7 "clang+llvm-18.1.7-aarch64-linux-gnu.tar.xz": "f0df4a38d4e205ee9dea23fdbe1b3acb0d3174d1366ef1488f1ea619cd6e6c0e", + "clang+llvm-18.1.7-armv7a-linux-gnueabihf.tar.gz": "ed5fb7aa5b66696e4a11a44531c94c2274cbfd92584dac50cbbbc4ed386594c4", "clang+llvm-18.1.7-powerpc64-ibm-aix-7.2.tar.xz": "a4317fe5bfc5579093f02bc6b63de3d1fa545ebca471405b70ff213d400e8da3", "clang+llvm-18.1.7-powerpc64le-linux-rhel-8.8.tar.xz": "8889adb5b6a6deffeaffd8c6dc0d2388c26660cb2c357df66b27561c7932ed66", "clang+llvm-18.1.7-x86_64-linux-gnu-ubuntu-18.04.tar.xz": "9aae4d652c438d8c44aaea480e52c7fd9b92f88f1c90673144165c7c8cbf9d28", @@ -510,6 +521,7 @@ _llvm_distributions = { # 18.1.8 "clang+llvm-18.1.8-aarch64-linux-gnu.tar.xz": "dcaa1bebbfbb86953fdfbdc7f938800229f75ad26c5c9375ef242edad737d999", "clang+llvm-18.1.8-arm64-apple-macos11.tar.xz": "4573b7f25f46d2a9c8882993f091c52f416c83271db6f5b213c93f0bd0346a10", + "clang+llvm-18.1.8-armv7a-linux-gnueabihf.tar.gz": "a4fc669dd54030f27e422fa67751509fa14bb90fbef32c2bd24c7f395c93c47c", "clang+llvm-18.1.8-powerpc64-ibm-aix-7.2.tar.xz": "0bf2df8cc823e1b76b2c42f5e8ac3ef1076865eee87a7098deb227d0f66b7e7c", "clang+llvm-18.1.8-powerpc64le-linux-rhel-8.8.tar.xz": "b3df0c1607bfb04fe268c2e80542aba6e63ef0766a0bc4100ccf6a1ea99a0a1b", "clang+llvm-18.1.8-x86_64-linux-gnu-ubuntu-18.04.tar.xz": "54ec30358afcc9fb8aa74307db3046f5187f9fb89fb37064cdde906e062ebf36", @@ -517,38 +529,51 @@ _llvm_distributions = { # 19.1.0 "LLVM-19.1.0-Linux-X64.tar.xz": "cee77d641690466a193d9b88c89705de1c02bbad46bde6a3b126793c0a0f2923", + "LLVM-19.1.0-Windows-X64.tar.xz": "a132377865d72bc7452343d59d05da63266ffc928b4072d63fb854fd42097dc4", "LLVM-19.1.0-macOS-ARM64.tar.xz": "9da86f64a99f5ce9b679caf54e938736ca269c5e069d0c94ad08b995c5f25c16", "LLVM-19.1.0-macOS-X64.tar.xz": "264f2f1e8b67f066749349ae8b4943d346cd44e099464164ef21b42a57663540", - "LLVM-19.1.0-Windows-X64.tar.xz": "a132377865d72bc7452343d59d05da63266ffc928b4072d63fb854fd42097dc4", + "clang+llvm-19.1.0-aarch64-linux-gnu.tar.xz": "7bb54afd330fe1a1c2d4c593fa1e2dbe2abd9bf34fb3597994ff41e443cf144b", + "clang+llvm-19.1.0-armv7a-linux-gnueabihf.tar.gz": "d2f9f7fe803b38dc2fa64a1b2d1d3356f230f9ba402c321d8615ba3598f5cb66", + "clang+llvm-19.1.0-x86_64-pc-windows-msvc.tar.xz": "de3199fcace428386878e5a98c2be35576459f140f405eddc8b1f8e91f5dae64", # 19.1.1 "LLVM-19.1.1-Linux-X64.tar.xz": "8204de000b6a6921f0572e038336601e3225898e9a253c8aaa43b0a5fae8a4ce", - "LLVM-19.1.1-macOS-ARM64.tar.xz": "e00def99a6b32de61fffabd4bb85290589731f3f5cb9836fd51770256cd694dd", "LLVM-19.1.1-Windows-X64.tar.xz": "dafd21646d10b7a59cf755002b608a657173c22daf63d99663eb193aedde48c1", + "LLVM-19.1.1-macOS-ARM64.tar.xz": "e00def99a6b32de61fffabd4bb85290589731f3f5cb9836fd51770256cd694dd", + "clang+llvm-19.1.1-aarch64-linux-gnu.tar.xz": "414d2ebef10c5035e9df10a224e81b484dbe17d319373050d0c1b3b1467040d2", + "clang+llvm-19.1.1-armv7a-linux-gnueabihf.tar.gz": "bf63f9092d1cd4c63d1858182ffa2a1713d4f00bc637d4169717bd5c9c905be3", + "clang+llvm-19.1.1-x86_64-pc-windows-msvc.tar.xz": "621fc299fceb1bbdae927e355d1073034c9a1bbdda5a46a27e217c56af72f72a", # 19.1.2 "LLVM-19.1.2-Linux-X64.tar.xz": "5b7fe5b2dbbacadd0fee17ac45103c0393bc4b5a9096506a865aa2fbcba976a7", - "LLVM-19.1.2-macOS-ARM64.tar.xz": "62eb2d8e9f610595fc53db020e26e4576c57c5853a55981292f17730916e676d", "LLVM-19.1.2-Windows-X64.tar.xz": "3aa49c72622c14caabb63f80cc156ce1d6806e12af554754ae1084bd7bc8f6ba", + "LLVM-19.1.2-macOS-ARM64.tar.xz": "62eb2d8e9f610595fc53db020e26e4576c57c5853a55981292f17730916e676d", + "clang+llvm-19.1.2-aarch64-linux-gnu.tar.xz": "eb9ab2b24f4b75f8010feed4a43d5a4ebf3c7e1ccff881e1cdf12a122748e7c4", + "clang+llvm-19.1.2-armv7a-linux-gnueabihf.tar.gz": "5a86ae82efca724882af265e3a8d7a37f09cf217483604882365b6ecb4195f00", + "clang+llvm-19.1.2-x86_64-pc-windows-msvc.tar.xz": "14e764eb79e4ed58da1b88320e33e5eb6c6064103446b47c4439b14292b99d12", # 19.1.3 "LLVM-19.1.3-Linux-X64.tar.xz": "052a5ee117782aab5893dba2cdf2cb97c3d873f7a50ba6b1690594161c75c519", + "LLVM-19.1.3-Windows-X64.tar.xz": "1077267ca353a1e236055ed4b57d6a404d09c40b01bd27dc882870395cdc1aae", "LLVM-19.1.3-macOS-ARM64.tar.xz": "80a54a467e9e770a76ba9670e89a235224ec47578cc4d4dbd928592813732518", "LLVM-19.1.3-macOS-X64.tar.xz": "52ea30f3089af4e086a98638a16167c5a20d253d43f7146c058e3e9e6d33274f", - "LLVM-19.1.3-Windows-X64.tar.xz": "1077267ca353a1e236055ed4b57d6a404d09c40b01bd27dc882870395cdc1aae", "clang+llvm-19.1.3-aarch64-linux-gnu.tar.xz": "a730175e58233f20a99ecab0015d8cd0f1af5d92411ca1f9e3e472645d889bcd", + "clang+llvm-19.1.3-armv7a-linux-gnueabihf.tar.gz": "b602416a0ea588da73d535050a7efc2b89bc58c69556cd2d828d413c258ba215", + "clang+llvm-19.1.3-x86_64-pc-windows-msvc.tar.xz": "84789dc852e67f8507861a5dea9ed41f11ad7a6c9d3db6d52f04d72b3e4e29d3", # 19.1.4 "LLVM-19.1.4-Linux-X64.tar.xz": "da7e0f571b440e5ef9ae6e061ae6afc1071179e18f86f77cf630dabbed11a5f6", "LLVM-19.1.4-macOS-ARM64.tar.xz": "52245bc374fdb9f3665046fe7319b5b8165ca2732053c74f06ba1e90e142ed8e", "LLVM-19.1.4-macOS-X64.tar.xz": "eedb896c193cc3bad35a9f132d91e16cf73d33723f35d63dcaae4755872674c2", "clang+llvm-19.1.4-aarch64-linux-gnu.tar.xz": "c42ea92e7a4cfad96b2b0d6c7872c6e9a9960a1d8a56c1847eca45d79cd67533", + "clang+llvm-19.1.4-armv7a-linux-gnueabihf.tar.gz": "1725f51a2be83feb9e7d2c393e179fa646f85eb80d17dca8b1c65bcee43455a4", "clang+llvm-19.1.4-x86_64-pc-windows-msvc.tar.xz": "5e965a1281c9df1fda8eddab3752ee6a3139e36ce469537f216cd938c498e6c3", # 19.1.5 "LLVM-19.1.5-Linux-X64.tar.xz": "13e9975b026d431c945927960e5f8c0a47a155a2f600f57e85f4d1482620c65f", "LLVM-19.1.5-macOS-X64.tar.xz": "f593d45992807d03c2aeb4c968c5cab9e78403430caea21dca4b787cbca3b9f4", "clang+llvm-19.1.5-aarch64-linux-gnu.tar.xz": "1bdc342b7d03cbcfafb2ffb8659eb0e4d5c6ddef6f56e0cad0e0c09c52577a4f", + "clang+llvm-19.1.5-armv7a-linux-gnueabihf.tar.gz": "f0058f9fc80dd939609a1ac2cbf791bbaf3e66ee56eb320b93f1494f3478cb57", "clang+llvm-19.1.5-x86_64-pc-windows-msvc.tar.xz": "467d1a73ca938f47734af3baac2e78c5e730285469096ee088bb5c9590cabd70", # 19.1.6 @@ -556,6 +581,7 @@ _llvm_distributions = { "LLVM-19.1.6-macOS-ARM64.tar.xz": "2c28bcd132ce3db367354c892839a962aa01b7b850a25e61316178f2ac72ecac", "LLVM-19.1.6-macOS-X64.tar.xz": "58ce29a2adb82872b6de49018091c6d844ca555a9b017faa698f6df409b25281", "clang+llvm-19.1.6-aarch64-linux-gnu.tar.xz": "f6fd8cf8bb12f507c4a55609ef6a435b3c59bc658008b712b80ec1cdc1ee9325", + "clang+llvm-19.1.6-armv7a-linux-gnueabihf.tar.gz": "3ce188e3394c2bf2d2f2ec1c63f4e450e10092d642953d1b73940cfe9213f9ba", "clang+llvm-19.1.6-x86_64-pc-windows-msvc.tar.xz": "d2e64d4d6eca9199ea5b8ac018e626fe2f2814ab90247c335fc9fd7448681bb3", # 19.1.7 @@ -563,6 +589,7 @@ _llvm_distributions = { "LLVM-19.1.7-macOS-ARM64.tar.xz": "d93bf12952d89fe4ec7501c40475718b722407da6a8d651f05c995863468e570", "LLVM-19.1.7-macOS-X64.tar.xz": "49405e75fbe7ad6f8139a33f59ec8c5112b75b3027405c7b92d19f4c6f02c78a", "clang+llvm-19.1.7-aarch64-linux-gnu.tar.xz": "a73d9326e5d756e3937df6a9f621664d76403b59119f741901106b387e53a6ae", + "clang+llvm-19.1.7-armv7a-linux-gnueabihf.tar.gz": "dedde2acbc164649b77d6f6635e8551218c9aed5a6df4c09b2614aaccc0c05b2", "clang+llvm-19.1.7-x86_64-pc-windows-msvc.tar.xz": "b4557b4f012161f56a2f5d9e877ab9635cafd7a08f7affe14829bd60c9d357f0", # 20.1.0 @@ -570,6 +597,7 @@ _llvm_distributions = { "LLVM-20.1.0-Linux-X64.tar.xz": "954ac51498519f6ed9540714fb04bc401f70039b296a8160dd1559be380788d7", "LLVM-20.1.0-macOS-ARM64.tar.xz": "2c42ec26ec50c4bf8b95585f762b9d2f5b385d170dee772d9c1d6c9a7190dcef", "clang+llvm-20.1.0-aarch64-pc-windows-msvc.tar.xz": "f52e40d68843ed6205858e817ed791295ef51e526037186352a1aeac4a59e51a", + "clang+llvm-20.1.0-armv7a-linux-gnueabihf.tar.gz": "487d38a49bd64ef03b46ce1dc6f32645052ded09f96ada847e4f46e69c799d01", "clang+llvm-20.1.0-x86_64-pc-windows-msvc.tar.xz": "91e29416f4a0c188368f0540a5538efc0d8a9f7134afba7a2160296472ce84eb", # 20.1.1 @@ -577,12 +605,15 @@ _llvm_distributions = { "LLVM-20.1.1-Linux-X64.tar.xz": "b1f40360adbf31934d5d3d999c5f91f7e52e089ae984d237565cc4c23bbfa283", "LLVM-20.1.1-macOS-ARM64.tar.xz": "ae52012b28bb43e1aa698aa347e37d06edb4643895b8bb189ed275025cd349ed", "clang+llvm-20.1.1-aarch64-pc-windows-msvc.tar.xz": "6ee4c1a8c51cf081e19a7225d802d160cc888cdc3a8da07dcbdb5768e3160244", + "clang+llvm-20.1.1-armv7a-linux-gnueabihf.tar.gz": "c443e13fc8293f688acdd1d715cb56cd36c763a0525b86265417d57cdfa42994", "clang+llvm-20.1.1-x86_64-pc-windows-msvc.tar.xz": "f8114cb674317e8a303731b1f9d22bf37b8c571b64f600abe528e92275ed4ace", # 20.1.2 "LLVM-20.1.2-Linux-ARM64.tar.xz": "41a6a2892cf66cd7c275753f2d1afe0e33b26c9674eff7d114fb36a52253436a", "LLVM-20.1.2-Linux-X64.tar.xz": "3a392f151375eeed4fd50c6b6f7c7203da37b373a57f220ae58ef62b8aade3cc", "LLVM-20.1.2-macOS-ARM64.tar.xz": "e502de0ccaa12dec9b7499c9e15e896006feda438078aba8b97894ae3218d4e3", + "clang+llvm-20.1.2-aarch64-pc-windows-msvc.tar.xz": "cb82f730a7d0d70866d4228fedafb8aca36d7dc3fd8a74a570f72ed95a52d5ed", + "clang+llvm-20.1.2-armv7a-linux-gnueabihf.tar.gz": "79ea2536f45a9f4c3fa89c3c03dae29be2e5fcd1bea2e163939ba527aa913219", "clang+llvm-20.1.2-x86_64-pc-windows-msvc.tar.xz": "8e771a685cd718303ea0d632a8a95ad7b3cb17068f3952fbefa64a77290324d8", # 20.1.3 @@ -590,7 +621,12 @@ _llvm_distributions = { "LLVM-20.1.3-Linux-X64.tar.xz": "c75103f520626cd2137a7e907998f12fff64136514ade1bb0a259995ae2de80e", "LLVM-20.1.3-macOS-ARM64.tar.xz": "70cd48fcd6b838690149bd00a85270d054c1c410d430f7c51f6d6e9019790d62", "LLVM-20.1.3-macOS-X64.tar.xz": "c3043862e4715ed3dc9f2c83e2a600e75ffecc005b977a391af50664a63fed2b", + "clang+llvm-20.1.3-aarch64-pc-windows-msvc.tar.xz": "fcbbd259fc4430f96f4a1ee51bc41038cd1e93138675d2f46baa2d479f0eb306", + "clang+llvm-20.1.3-armv7a-linux-gnueabihf.tar.gz": "fc6ebfbabbbfea74f164f7c19ae953bcf066d34d348c6ef758d32fb87872b9e3", "clang+llvm-20.1.3-x86_64-pc-windows-msvc.tar.xz": "3831e10ca8409e2288d70491c2fd925f5d5f9b644abf4553552887ff9ce32798", + + # Refer to variable declaration on how to update! + # Example update, run: utils/llvm_checksums.sh -g -v 15.0.6 } # Note: Unlike the user-specified llvm_mirror attribute, the URL prefixes in @@ -681,6 +717,82 @@ def _get_llvm_version(rctx): fail("LLVM version string missing for ({os}, {arch})", os = _os(rctx), arch = _arch(rctx)) return llvm_version +def _find_llvm_basename_list(llvm_version, arch, os): + """Lookup (llvm_version, arch, os) in the list of basenames in `_llvm_distributions.`""" + prefixes = [] + + if os == "raspbian": + # TODO: Check the reported `arch` and return [] if unsupported + prefixes.append("clang+llvm-{llvm_version}-{arch}-{os}".format( + llvm_version = llvm_version, + arch = "armv7a", + os = "linux-gnueabihf", + )) + elif arch == "x86_64" and os == "pc-solaris2.11": + # TODO: Check the reported `arch` and return [] if unsupported + prefixes.append("clang+llvm-{llvm_version}-{arch}-{os}".format( + llvm_version = llvm_version, + arch = "amd64", + os = os, + )) + else: + llvm_new_arch = { + "aarch64": "ARM64", + "x86_64": "X64", + }.get(arch, arch) + llvm_new_os = { + "darwin": "macOS", + "linux": "Linux", + "windows": "Windows", + }.get(os, os) + llvm_old_os = { + "darwin": "apple-darwin", + "linux": "linux-gnu", + "windows": "pc-windows-msvc", + }.get(os, os) + + prefixes.append("LLVM-{llvm_version}-{os}-{arch}".format( + llvm_version = llvm_version, + arch = llvm_new_arch, + os = llvm_new_os, + )) + if arch in ["powerpc64", "powerpc64le", "sparcv9"]: + prefixes.append("clang+llvm-{llvm_version}-{arch}-".format( + llvm_version = llvm_version, + arch = arch, + )) + elif arch in ["aarch64"] and os in ["darwin"]: + prefixes.append("clang+llvm-{llvm_version}-{arch}-{os}".format( + llvm_version = llvm_version, + arch = "arm64", + os = "apple-darwin", + )) + prefixes.append("clang+llvm-{llvm_version}-{arch}-{os}".format( + llvm_version = llvm_version, + arch = "arm64", + os = "apple-macos", + )) + else: + prefixes.append("clang+llvm-{llvm_version}-{arch}-{os}".format( + llvm_version = llvm_version, + arch = arch, + os = llvm_old_os, + )) + + suffixes = [".tar.gz", ".tar.xz"] + for prefix in prefixes: + for suffix in suffixes: + basename = prefix + suffix + if basename in _llvm_distributions: + return [basename] + + basenames = [] + for dist in _llvm_distributions: + for prefix in prefixes: + if dist.startswith(prefix): + basenames.append(dist) + return basenames + def _distribution_urls(rctx): llvm_version = _get_llvm_version(rctx) @@ -714,3 +826,93 @@ def _distribution_urls(rctx): strip_prefix = strip_prefix.rstrip("-rhel86") return urls, sha256, strip_prefix + +DistrubutionsInfo = provider( + "The output info for the `write_distributions` rule.", + fields = { + "out": "output File", + }, +) + +def _version_ge(lhs, rhs): + _lhs = lhs.split(".") + _rhs = rhs.split(".") + for p in range(3): + if int(_lhs[p]) < int(_rhs[p]): + return False + return len(_lhs) >= len(_rhs) + +def _write_distributions_impl(ctx): + """Analyze the configured versions and write to a file for test consumption. + + The test generated file '.out' contains the following lines: + - a 'del:' line denotes a llvm distribution basename that was not found. + - a 'add:' line denotes a version that was predicted but does not exist. + + Lines of type `add:` should never occur as the algorithm is supposed to + verify that predicted distributions have been configured. Otherwise the + algorithm could not know the hash value. + """ + arch_list = ["aarch64", "powerpc64", "powerpc64le", "sparcv9", "x86_64"] + os_list = [ + "darwin", + "linux", + "raspbian", + "pc-solaris2.11", # TODO: No clue how this one works + "windows", + ] + + MIN_VERSION = "18.0.0" + version_list = [] + for name in _llvm_distributions.keys(): + for prefix in ["LLVM-", "clang+llvm-"]: + name = name.removeprefix(prefix) + version = name.split("-", 1)[0] + if not _version_ge(version, MIN_VERSION): + continue + version_list.append(version) + for version in _llvm_distributions_base_url.keys(): + if not _version_ge(version, MIN_VERSION): + continue + version_list.append(version) + versions = set(version_list) + + output = [] + for version in versions: + output.append("version: " + version) + + not_found = { + k: v for k, v in _llvm_distributions.items() + if _version_ge(k.split("-")[1], MIN_VERSION) + } + result = {} + + for version in versions: + if not _version_ge(version, MIN_VERSION): + continue + for arch in arch_list: + for os in os_list: + basenames = _find_llvm_basename_list(version, arch, os) + if len(basenames) != 1: + continue + basename = basenames[0] + if basename in _llvm_distributions: + if basename in not_found: + not_found.pop(basename) + else: + result[basename] = True + for dist in not_found: + result[dist] = False + output += [("add: " if found else "del: ") + dist for dist, found in result.items()] + out = ctx.actions.declare_file(ctx.label.name + ".out") + ctx.actions.write(out, "\n".join(output) + "\n") + return [ + DefaultInfo(files = depset([out])), + DistrubutionsInfo(out = out), + ] + +write_distributions = rule( + implementation = _write_distributions_impl, + output_to_genfiles = True, + provides = [DefaultInfo, DistrubutionsInfo], +) diff --git a/toolchain/internal/llvm_distributions.golden.txt b/toolchain/internal/llvm_distributions.golden.txt new file mode 100644 index 000000000..059848df1 --- /dev/null +++ b/toolchain/internal/llvm_distributions.golden.txt @@ -0,0 +1,27 @@ +version: 18.1.0 +version: 18.1.1 +version: 18.1.2 +version: 18.1.3 +version: 18.1.4 +version: 18.1.5 +version: 18.1.6 +version: 18.1.7 +version: 18.1.8 +version: 19.1.0 +version: 19.1.1 +version: 19.1.2 +version: 19.1.3 +version: 19.1.4 +version: 19.1.5 +version: 19.1.6 +version: 19.1.7 +version: 20.1.0 +version: 20.1.1 +version: 20.1.2 +version: 20.1.3 +del: clang+llvm-18.1.0-sparcv9-sun-solaris2.11.tar.xz +del: clang+llvm-18.1.0-sparcv9-unknown-linux-gnu.tar.xz +del: clang+llvm-19.1.0-x86_64-pc-windows-msvc.tar.xz +del: clang+llvm-19.1.1-x86_64-pc-windows-msvc.tar.xz +del: clang+llvm-19.1.2-x86_64-pc-windows-msvc.tar.xz +del: clang+llvm-19.1.3-x86_64-pc-windows-msvc.tar.xz diff --git a/utils/llvm_checksums.sh b/utils/llvm_checksums.sh index deac4d49b..ebde827c2 100755 --- a/utils/llvm_checksums.sh +++ b/utils/llvm_checksums.sh @@ -16,15 +16,18 @@ set -euo pipefail use_github_host=0 +tmp_dir= -while getopts "v:gh" opt; do +while getopts "t:v:gh" opt; do case "${opt}" in + "t") tmp_dir="${OPTARG}" ;; "v") llvm_version="${OPTARG}" ;; "g") use_github_host=1 ;; "h") echo "Usage:" - echo "-v - Version of clang+llvm to use" - echo "-g - Use github to download releases" + echo "-t - Optional: Specify a temp directory to download distributions to." + echo "-v - Version of clang+llvm to use." + echo "-g - Use github to download releases." exit 2 ;; *) @@ -35,36 +38,42 @@ while getopts "v:gh" opt; do done if [[ -z ${llvm_version-} ]]; then - echo "Usage: ${BASH_SOURCE[0]} [-g] -v llvm_version" + echo "Usage: ${BASH_SOURCE[0]} [-t ] [-g] -v " exit 1 fi -tmp_dir="$(mktemp -d)" - cleanup() { rc=$? rm -rf "${tmp_dir}" exit "${rc}" } -trap 'cleanup' INT HUP QUIT TERM EXIT + +if [[ -z "${tmp_dir}" ]]; then + tmp_dir="$(mktemp -d)" + echo "Using temp dir: '${tmp_dir}'" + trap 'cleanup' INT HUP QUIT TERM EXIT +elif [[ ! -r "${tmp_dir}" ]]; then + echo "Temp directory does not exist: '${tmp_dir}'." + exit 2 +fi llvm_host() { local url_base="releases.llvm.org/${llvm_version}" output_dir="${tmp_dir}/${url_base}" wget --recursive --level 1 --directory-prefix="${tmp_dir}" \ - --accept-regex "(clang%2bllvm|LLVM)-.*tar.xz$" "http://${url_base}/" + --accept-regex "(clang%2bllvm|LLVM)-.*tar.(xz|gz)$" "http://${url_base}/" } github_host() { - output_dir="${tmp_dir}" + output_dir="${tmp_dir}/${llvm_version}" + mkdir -p "${output_dir}" ( cd "${output_dir}" curl -s "https://api.github.com/repos/llvm/llvm-project/releases/tags/llvmorg-${llvm_version}" | - jq .assets[].browser_download_url | - tee ./urls.txt | - grep -E '(clang%2Bllvm|LLVM)-.*tar.xz"$' | + tee ./releases.json | + jq '.assets[]|select(any(.name; test("^(clang[+]llvm|LLVM)-.*tar.(xz|gz)$")))|.browser_download_url' | tee ./filtered_urls.txt | - xargs -n1 curl -L -O + xargs -n1 curl -L -O -C - ) } @@ -76,8 +85,8 @@ fi echo "" echo "====" -echo "Checksums for clang+llvm distributions are:" -find "${output_dir}" -type f -name '*.xz' -exec shasum -a 256 {} \; | +echo "Checksums for clang+llvm distributions are (${output_dir}):" +find "${output_dir}" -type f \( -name 'clang%2?llvm-*.tar.*' -o -name 'LLVM-*.tar.*' \) \( -name '*.gz' -o -name '*.xz' \) -exec shasum -a 256 {} \; | sed -e "s@${output_dir}/@@" | awk '{ printf "\"%s\": \"%s\",\n", $2, $1 }' | sed -e 's/%2[Bb]/+/' | From 913e036b77941e6013e3de844b7e88014128653d Mon Sep 17 00:00:00 2001 From: helly25 Date: Fri, 25 Apr 2025 20:13:21 +0000 Subject: [PATCH 02/10] Split `llvm_release_name` into the part for version 19+ and earlier. For version 19+ determine which entries can be found and which ones cannot. --- toolchain/internal/llvm_distributions.bzl | 84 ++----------------- .../internal/llvm_distributions.golden.txt | 42 ++++++---- toolchain/internal/release_name.bzl | 39 +++++---- 3 files changed, 55 insertions(+), 110 deletions(-) diff --git a/toolchain/internal/llvm_distributions.bzl b/toolchain/internal/llvm_distributions.bzl index d8bb93312..e21c8c8be 100644 --- a/toolchain/internal/llvm_distributions.bzl +++ b/toolchain/internal/llvm_distributions.bzl @@ -14,7 +14,7 @@ load("@bazel_tools//tools/build_defs/repo:utils.bzl", "read_netrc", "use_netrc") load("//toolchain/internal:common.bzl", _arch = "arch", _attr_dict = "attr_dict", _exec_os_arch_dict_value = "exec_os_arch_dict_value", _os = "os") -load("//toolchain/internal:release_name.bzl", _llvm_release_name = "llvm_release_name") +load("//toolchain/internal:release_name.bzl", _llvm_release_name = "llvm_release_name", _llvm_release_name_19 = "llvm_release_name_19") # If a new LLVM version is missing from this list, please add the shasums here # and send a PR on github. To compute the shasum block, you can run (for example): @@ -719,79 +719,10 @@ def _get_llvm_version(rctx): def _find_llvm_basename_list(llvm_version, arch, os): """Lookup (llvm_version, arch, os) in the list of basenames in `_llvm_distributions.`""" - prefixes = [] - - if os == "raspbian": - # TODO: Check the reported `arch` and return [] if unsupported - prefixes.append("clang+llvm-{llvm_version}-{arch}-{os}".format( - llvm_version = llvm_version, - arch = "armv7a", - os = "linux-gnueabihf", - )) - elif arch == "x86_64" and os == "pc-solaris2.11": - # TODO: Check the reported `arch` and return [] if unsupported - prefixes.append("clang+llvm-{llvm_version}-{arch}-{os}".format( - llvm_version = llvm_version, - arch = "amd64", - os = os, - )) - else: - llvm_new_arch = { - "aarch64": "ARM64", - "x86_64": "X64", - }.get(arch, arch) - llvm_new_os = { - "darwin": "macOS", - "linux": "Linux", - "windows": "Windows", - }.get(os, os) - llvm_old_os = { - "darwin": "apple-darwin", - "linux": "linux-gnu", - "windows": "pc-windows-msvc", - }.get(os, os) - - prefixes.append("LLVM-{llvm_version}-{os}-{arch}".format( - llvm_version = llvm_version, - arch = llvm_new_arch, - os = llvm_new_os, - )) - if arch in ["powerpc64", "powerpc64le", "sparcv9"]: - prefixes.append("clang+llvm-{llvm_version}-{arch}-".format( - llvm_version = llvm_version, - arch = arch, - )) - elif arch in ["aarch64"] and os in ["darwin"]: - prefixes.append("clang+llvm-{llvm_version}-{arch}-{os}".format( - llvm_version = llvm_version, - arch = "arm64", - os = "apple-darwin", - )) - prefixes.append("clang+llvm-{llvm_version}-{arch}-{os}".format( - llvm_version = llvm_version, - arch = "arm64", - os = "apple-macos", - )) - else: - prefixes.append("clang+llvm-{llvm_version}-{arch}-{os}".format( - llvm_version = llvm_version, - arch = arch, - os = llvm_old_os, - )) - - suffixes = [".tar.gz", ".tar.xz"] - for prefix in prefixes: - for suffix in suffixes: - basename = prefix + suffix - if basename in _llvm_distributions: - return [basename] - - basenames = [] - for dist in _llvm_distributions: - for prefix in prefixes: - if dist.startswith(prefix): - basenames.append(dist) - return basenames + name = _llvm_release_name_19(llvm_version, arch, os) + if name in _llvm_distributions: + return [name] + return [] def _distribution_urls(rctx): llvm_version = _get_llvm_version(rctx) @@ -853,16 +784,15 @@ def _write_distributions_impl(ctx): verify that predicted distributions have been configured. Otherwise the algorithm could not know the hash value. """ - arch_list = ["aarch64", "powerpc64", "powerpc64le", "sparcv9", "x86_64"] + arch_list = ["aarch64", "x86_64"] os_list = [ "darwin", "linux", "raspbian", - "pc-solaris2.11", # TODO: No clue how this one works "windows", ] - MIN_VERSION = "18.0.0" + MIN_VERSION = "19.0.0" version_list = [] for name in _llvm_distributions.keys(): for prefix in ["LLVM-", "clang+llvm-"]: diff --git a/toolchain/internal/llvm_distributions.golden.txt b/toolchain/internal/llvm_distributions.golden.txt index 059848df1..0b67dafa0 100644 --- a/toolchain/internal/llvm_distributions.golden.txt +++ b/toolchain/internal/llvm_distributions.golden.txt @@ -1,12 +1,3 @@ -version: 18.1.0 -version: 18.1.1 -version: 18.1.2 -version: 18.1.3 -version: 18.1.4 -version: 18.1.5 -version: 18.1.6 -version: 18.1.7 -version: 18.1.8 version: 19.1.0 version: 19.1.1 version: 19.1.2 @@ -19,9 +10,30 @@ version: 20.1.0 version: 20.1.1 version: 20.1.2 version: 20.1.3 -del: clang+llvm-18.1.0-sparcv9-sun-solaris2.11.tar.xz -del: clang+llvm-18.1.0-sparcv9-unknown-linux-gnu.tar.xz -del: clang+llvm-19.1.0-x86_64-pc-windows-msvc.tar.xz -del: clang+llvm-19.1.1-x86_64-pc-windows-msvc.tar.xz -del: clang+llvm-19.1.2-x86_64-pc-windows-msvc.tar.xz -del: clang+llvm-19.1.3-x86_64-pc-windows-msvc.tar.xz +del: clang+llvm-19.1.0-aarch64-linux-gnu.tar.xz +del: clang+llvm-19.1.0-armv7a-linux-gnueabihf.tar.gz +del: clang+llvm-19.1.1-aarch64-linux-gnu.tar.xz +del: clang+llvm-19.1.1-armv7a-linux-gnueabihf.tar.gz +del: clang+llvm-19.1.2-aarch64-linux-gnu.tar.xz +del: clang+llvm-19.1.2-armv7a-linux-gnueabihf.tar.gz +del: clang+llvm-19.1.3-armv7a-linux-gnueabihf.tar.gz +del: clang+llvm-19.1.4-armv7a-linux-gnueabihf.tar.gz +del: clang+llvm-19.1.4-x86_64-pc-windows-msvc.tar.xz +del: clang+llvm-19.1.5-armv7a-linux-gnueabihf.tar.gz +del: clang+llvm-19.1.5-x86_64-pc-windows-msvc.tar.xz +del: clang+llvm-19.1.6-armv7a-linux-gnueabihf.tar.gz +del: clang+llvm-19.1.6-x86_64-pc-windows-msvc.tar.xz +del: clang+llvm-19.1.7-armv7a-linux-gnueabihf.tar.gz +del: clang+llvm-19.1.7-x86_64-pc-windows-msvc.tar.xz +del: clang+llvm-20.1.0-aarch64-pc-windows-msvc.tar.xz +del: clang+llvm-20.1.0-armv7a-linux-gnueabihf.tar.gz +del: clang+llvm-20.1.0-x86_64-pc-windows-msvc.tar.xz +del: clang+llvm-20.1.1-aarch64-pc-windows-msvc.tar.xz +del: clang+llvm-20.1.1-armv7a-linux-gnueabihf.tar.gz +del: clang+llvm-20.1.1-x86_64-pc-windows-msvc.tar.xz +del: clang+llvm-20.1.2-aarch64-pc-windows-msvc.tar.xz +del: clang+llvm-20.1.2-armv7a-linux-gnueabihf.tar.gz +del: clang+llvm-20.1.2-x86_64-pc-windows-msvc.tar.xz +del: clang+llvm-20.1.3-aarch64-pc-windows-msvc.tar.xz +del: clang+llvm-20.1.3-armv7a-linux-gnueabihf.tar.gz +del: clang+llvm-20.1.3-x86_64-pc-windows-msvc.tar.xz diff --git a/toolchain/internal/release_name.bzl b/toolchain/internal/release_name.bzl index a17d41687..fec08d66c 100755 --- a/toolchain/internal/release_name.bzl +++ b/toolchain/internal/release_name.bzl @@ -211,27 +211,30 @@ def _resolve_version_for_suse(major_llvm_version, llvm_version): os_name = _ubuntu_osname("x86_64", "20.04", major_llvm_version, llvm_version) return os_name +def llvm_release_name_19(llvm_version, rctx_arch, rctx_os): + arch = { + "aarch64": "ARM64", + "x86_64": "X64", + }.get(rctx_arch, rctx_arch) + os = { + "darwin": "macOS", + "linux": "Linux", + "windows": "Windows", + }.get(rctx_os, rctx_os) + if arch == "ARM64" and os == "Linux" and llvm_version in ["19.1.7", "19.1.6", "19.1.5", "19.1.4", "19.1.3"]: + return "clang+llvm-{llvm_version}-aarch64-linux-gnu.tar.xz".format( + llvm_version = llvm_version, + ) + return "LLVM-{llvm_version}-{os}-{arch}.tar.xz".format( + llvm_version = llvm_version, + arch = arch, + os = os, + ) + def llvm_release_name(rctx, llvm_version): major_llvm_version = _major_llvm_version(llvm_version) if major_llvm_version >= 19: - arch = { - "aarch64": "ARM64", - "x86_64": "X64", - }[_arch(rctx)] - os = { - "darwin": "macOS", - "linux": "Linux", - "windows": "Windows", - }[_os(rctx)] - if arch == "ARM64" and os == "Linux" and llvm_version in ["19.1.7", "19.1.6", "19.1.5", "19.1.4", "19.1.3"]: - return "clang+llvm-{llvm_version}-aarch64-linux-gnu.tar.xz".format( - llvm_version = llvm_version, - ) - return "LLVM-{llvm_version}-{os}-{arch}.tar.xz".format( - llvm_version = llvm_version, - arch = arch, - os = os, - ) + return llvm_release_name_19(llvm_version, _arch(rctx), _os(rctx)) else: (os, version, arch) = _os_version_arch(rctx) if os == "darwin": From 1bab45a28c8150209c7a04b42bd1ab9fbcf23167 Mon Sep 17 00:00:00 2001 From: helly25 Date: Fri, 25 Apr 2025 20:24:15 +0000 Subject: [PATCH 03/10] More non found versions. Documentation. --- toolchain/internal/llvm_distributions.bzl | 14 +++++++++++--- toolchain/internal/llvm_distributions.golden.txt | 4 ++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/toolchain/internal/llvm_distributions.bzl b/toolchain/internal/llvm_distributions.bzl index e21c8c8be..8e4610492 100644 --- a/toolchain/internal/llvm_distributions.bzl +++ b/toolchain/internal/llvm_distributions.bzl @@ -792,6 +792,7 @@ def _write_distributions_impl(ctx): "windows", ] + # Compute all unique version strings starting with `MIN_VERSION`. MIN_VERSION = "19.0.0" version_list = [] for name in _llvm_distributions.keys(): @@ -807,19 +808,24 @@ def _write_distributions_impl(ctx): version_list.append(version) versions = set(version_list) + # Write versions to output to check which versions we take into account. output = [] for version in versions: output.append("version: " + version) + # We keep track of versions in `not_found` and remove the ones we found. + # So at the end all version that were not found remain, hence the name. not_found = { k: v for k, v in _llvm_distributions.items() if _version_ge(k.split("-")[1], MIN_VERSION) } + + # While computing we add predicted versions that are not configured as True. + # At the end we add the not-found versions as False. result = {} - + + # For all versions X arch X os check if we can compute the distribution. for version in versions: - if not _version_ge(version, MIN_VERSION): - continue for arch in arch_list: for os in os_list: basenames = _find_llvm_basename_list(version, arch, os) @@ -831,6 +837,8 @@ def _write_distributions_impl(ctx): not_found.pop(basename) else: result[basename] = True + + # Build result for dist in not_found: result[dist] = False output += [("add: " if found else "del: ") + dist for dist, found in result.items()] diff --git a/toolchain/internal/llvm_distributions.golden.txt b/toolchain/internal/llvm_distributions.golden.txt index 0b67dafa0..13ebd1e09 100644 --- a/toolchain/internal/llvm_distributions.golden.txt +++ b/toolchain/internal/llvm_distributions.golden.txt @@ -12,11 +12,15 @@ version: 20.1.2 version: 20.1.3 del: clang+llvm-19.1.0-aarch64-linux-gnu.tar.xz del: clang+llvm-19.1.0-armv7a-linux-gnueabihf.tar.gz +del: clang+llvm-19.1.0-x86_64-pc-windows-msvc.tar.xz del: clang+llvm-19.1.1-aarch64-linux-gnu.tar.xz del: clang+llvm-19.1.1-armv7a-linux-gnueabihf.tar.gz +del: clang+llvm-19.1.1-x86_64-pc-windows-msvc.tar.xz del: clang+llvm-19.1.2-aarch64-linux-gnu.tar.xz del: clang+llvm-19.1.2-armv7a-linux-gnueabihf.tar.gz +del: clang+llvm-19.1.2-x86_64-pc-windows-msvc.tar.xz del: clang+llvm-19.1.3-armv7a-linux-gnueabihf.tar.gz +del: clang+llvm-19.1.3-x86_64-pc-windows-msvc.tar.xz del: clang+llvm-19.1.4-armv7a-linux-gnueabihf.tar.gz del: clang+llvm-19.1.4-x86_64-pc-windows-msvc.tar.xz del: clang+llvm-19.1.5-armv7a-linux-gnueabihf.tar.gz From 256f1c28952569a0528437ec8ae2a1fd6d523338 Mon Sep 17 00:00:00 2001 From: helly25 Date: Fri, 25 Apr 2025 20:28:12 +0000 Subject: [PATCH 04/10] format --- toolchain/internal/llvm_distributions.bzl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/toolchain/internal/llvm_distributions.bzl b/toolchain/internal/llvm_distributions.bzl index 8e4610492..61d6fc6d2 100644 --- a/toolchain/internal/llvm_distributions.bzl +++ b/toolchain/internal/llvm_distributions.bzl @@ -816,7 +816,8 @@ def _write_distributions_impl(ctx): # We keep track of versions in `not_found` and remove the ones we found. # So at the end all version that were not found remain, hence the name. not_found = { - k: v for k, v in _llvm_distributions.items() + k: v + for k, v in _llvm_distributions.items() if _version_ge(k.split("-")[1], MIN_VERSION) } From 584682022cd6baa1c5992337c157405bd5b7ca28 Mon Sep 17 00:00:00 2001 From: helly25 Date: Fri, 25 Apr 2025 20:32:09 +0000 Subject: [PATCH 05/10] Emulate set with dict for older bazel versions. --- toolchain/internal/llvm_distributions.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/toolchain/internal/llvm_distributions.bzl b/toolchain/internal/llvm_distributions.bzl index 61d6fc6d2..4dad84491 100644 --- a/toolchain/internal/llvm_distributions.bzl +++ b/toolchain/internal/llvm_distributions.bzl @@ -806,7 +806,7 @@ def _write_distributions_impl(ctx): if not _version_ge(version, MIN_VERSION): continue version_list.append(version) - versions = set(version_list) + versions = {v: v for v in version_list} # Write versions to output to check which versions we take into account. output = [] @@ -826,7 +826,7 @@ def _write_distributions_impl(ctx): result = {} # For all versions X arch X os check if we can compute the distribution. - for version in versions: + for version in versions.keys(): for arch in arch_list: for os in os_list: basenames = _find_llvm_basename_list(version, arch, os) From f185c6d7b8c4b22c35abfef3d87113afcfedf5d1 Mon Sep 17 00:00:00 2001 From: helly25 Date: Fri, 25 Apr 2025 20:50:16 +0000 Subject: [PATCH 06/10] Add version test to CI --- .github/workflows/tests.yml | 9 ++++++ tests/scripts/run_toolchain_tests.sh | 45 ++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 tests/scripts/run_toolchain_tests.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 26614fbd2..559351590 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -41,6 +41,15 @@ jobs: USE_BAZEL_VERSION: ${{ matrix.bazel_version }} USE_BZLMOD: ${{ matrix.bzlmod }} run: tests/scripts/run_tests.sh + toolchain_test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Test + env: + USE_BAZEL_VERSION: latest + USE_BZLMOD: true + run: tests/scripts/run_toolchain_tests.sh external_test: strategy: fail-fast: false diff --git a/tests/scripts/run_toolchain_tests.sh b/tests/scripts/run_toolchain_tests.sh new file mode 100644 index 000000000..1bc7cac24 --- /dev/null +++ b/tests/scripts/run_toolchain_tests.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright 2018 The Bazel Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +while getopts "h" opt; do + case "${opt}" in + "h") + echo "Usage: No options" + exit 2 + ;; + *) + echo "invalid option: -${OPTARG}" + exit 1 + ;; + esac +done + +scripts_dir="$(dirname "${BASH_SOURCE[0]}")" +source "${scripts_dir}/bazel.sh" +"${bazel}" version + +set -x +test_args=( + "--check_direct_dependencies=off" +) + +targets=( + "//toolchain/..." +) + +"${bazel}" ${TEST_MIGRATION:+"--strict"} --bazelrc=/dev/null test \ + "${common_test_args[@]}" "${test_args[@]}" "${targets[@]}" From aac9b97ce13897e160750167e1133187abb430e6 Mon Sep 17 00:00:00 2001 From: helly25 Date: Fri, 25 Apr 2025 20:51:46 +0000 Subject: [PATCH 07/10] Mode +x --- tests/scripts/run_toolchain_tests.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tests/scripts/run_toolchain_tests.sh diff --git a/tests/scripts/run_toolchain_tests.sh b/tests/scripts/run_toolchain_tests.sh old mode 100644 new mode 100755 From e21fc6e905b1f5157c0a5be364d59ce33f6ebd97 Mon Sep 17 00:00:00 2001 From: helly25 Date: Sat, 26 Apr 2025 07:04:26 +0000 Subject: [PATCH 08/10] Address review comments. --- toolchain/internal/llvm_distributions.bzl | 25 ++++++----------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/toolchain/internal/llvm_distributions.bzl b/toolchain/internal/llvm_distributions.bzl index 4dad84491..07b1257b3 100644 --- a/toolchain/internal/llvm_distributions.bzl +++ b/toolchain/internal/llvm_distributions.bzl @@ -17,7 +17,8 @@ load("//toolchain/internal:common.bzl", _arch = "arch", _attr_dict = "attr_dict" load("//toolchain/internal:release_name.bzl", _llvm_release_name = "llvm_release_name", _llvm_release_name_19 = "llvm_release_name_19") # If a new LLVM version is missing from this list, please add the shasums here -# and send a PR on github. To compute the shasum block, you can run (for example): +# and the new version in toolchain/internal/llvm_distributions.golden.txt. +# Then send a PR on github. To compute the shasum block, you can run (for example): # utils/llvm_checksums.sh -g -v 15.0.6 # # To find all available release versions, search for "tag_name" in @@ -758,20 +759,11 @@ def _distribution_urls(rctx): return urls, sha256, strip_prefix -DistrubutionsInfo = provider( - "The output info for the `write_distributions` rule.", - fields = { - "out": "output File", - }, -) +def _parse_version(v): + return tuple([int(s) for s in v.split(".")]) def _version_ge(lhs, rhs): - _lhs = lhs.split(".") - _rhs = rhs.split(".") - for p in range(3): - if int(_lhs[p]) < int(_rhs[p]): - return False - return len(_lhs) >= len(_rhs) + return _parse_version(lhs) >= _parse_version(rhs) def _write_distributions_impl(ctx): """Analyze the configured versions and write to a file for test consumption. @@ -845,13 +837,8 @@ def _write_distributions_impl(ctx): output += [("add: " if found else "del: ") + dist for dist, found in result.items()] out = ctx.actions.declare_file(ctx.label.name + ".out") ctx.actions.write(out, "\n".join(output) + "\n") - return [ - DefaultInfo(files = depset([out])), - DistrubutionsInfo(out = out), - ] + return [DefaultInfo(files = depset([out]))] write_distributions = rule( implementation = _write_distributions_impl, - output_to_genfiles = True, - provides = [DefaultInfo, DistrubutionsInfo], ) From 45a5e5d1748beeebb0aa8eb418da50b11f9ae5b9 Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Sat, 26 Apr 2025 09:34:36 +0200 Subject: [PATCH 09/10] Update toolchain/internal/llvm_distributions.bzl --- toolchain/internal/llvm_distributions.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolchain/internal/llvm_distributions.bzl b/toolchain/internal/llvm_distributions.bzl index 07b1257b3..21cb20833 100644 --- a/toolchain/internal/llvm_distributions.bzl +++ b/toolchain/internal/llvm_distributions.bzl @@ -798,7 +798,7 @@ def _write_distributions_impl(ctx): if not _version_ge(version, MIN_VERSION): continue version_list.append(version) - versions = {v: v for v in version_list} + versions = {v: None for v in version_list} # Write versions to output to check which versions we take into account. output = [] From f087e27ea1a3f8e08d21be588a6600604031b08c Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Sat, 26 Apr 2025 09:34:55 +0200 Subject: [PATCH 10/10] Update toolchain/internal/llvm_distributions.bzl --- toolchain/internal/llvm_distributions.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolchain/internal/llvm_distributions.bzl b/toolchain/internal/llvm_distributions.bzl index 21cb20833..6acb27a46 100644 --- a/toolchain/internal/llvm_distributions.bzl +++ b/toolchain/internal/llvm_distributions.bzl @@ -802,7 +802,7 @@ def _write_distributions_impl(ctx): # Write versions to output to check which versions we take into account. output = [] - for version in versions: + for version in versions.keys(): output.append("version: " + version) # We keep track of versions in `not_found` and remove the ones we found.