From 3f6c0298fe6148b5b20842facd7d251779b23990 Mon Sep 17 00:00:00 2001 From: "stonezdj(Daojun Zhang)" Date: Wed, 15 Nov 2023 11:00:39 +0800 Subject: [PATCH 01/11] Add oidc provider name to systeminfo API (#19575) fixes #13198 Signed-off-by: stonezdj --- api/v2.0/swagger.yaml | 5 +++++ src/controller/systeminfo/controller.go | 10 +++++++++ src/controller/systeminfo/controller_test.go | 22 ++++++++++++++++++++ src/server/v2.0/handler/systeminfo.go | 1 + 4 files changed, 38 insertions(+) diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index ba4dcd770db..37e8564eb67 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -7867,6 +7867,11 @@ definitions: x-nullable: true x-omitempty: true $ref: '#/definitions/AuthproxySetting' + oidc_provider_name: + type: string + x-nullable: true + x-omitempty: true + description: The OIDC provider name, empty if current auth is not OIDC_auth or OIDC provider is not configured. AuthproxySetting: type: object properties: diff --git a/src/controller/systeminfo/controller.go b/src/controller/systeminfo/controller.go index 1487f036bb7..3e1aa93e70f 100644 --- a/src/controller/systeminfo/controller.go +++ b/src/controller/systeminfo/controller.go @@ -50,6 +50,7 @@ type Data struct { BannerMessage string AuthProxySettings *models.HTTPAuthProxy Protected *protectedData + OIDCProviderName string } type protectedData struct { @@ -103,6 +104,7 @@ func (c *controller) GetInfo(ctx context.Context, opt Options) (*Data, error) { SelfRegistration: utils.SafeCastBool(cfg[common.SelfRegistration]), HarborVersion: fmt.Sprintf("%s-%s", version.ReleaseVersion, version.GitCommit), BannerMessage: utils.SafeCastString(mgr.Get(ctx, common.BannerMessage).GetString()), + OIDCProviderName: OIDCProviderName(cfg), } if res.AuthMode == common.HTTPAuth { if s, err := config.HTTPAuthProxySetting(ctx); err == nil { @@ -137,6 +139,14 @@ func (c *controller) GetInfo(ctx context.Context, opt Options) (*Data, error) { return res, nil } +func OIDCProviderName(cfg map[string]interface{}) string { + authMode := utils.SafeCastString(cfg[common.AUTHMode]) + if authMode != common.OIDCAuth { + return "" + } + return utils.SafeCastString(cfg[common.OIDCName]) +} + func (c *controller) GetCapacity(ctx context.Context) (*imagestorage.Capacity, error) { systeminfo.Init() return imagestorage.GlobalDriver.Cap() diff --git a/src/controller/systeminfo/controller_test.go b/src/controller/systeminfo/controller_test.go index a2e98a1ba74..864d43a99d6 100644 --- a/src/controller/systeminfo/controller_test.go +++ b/src/controller/systeminfo/controller_test.go @@ -105,3 +105,25 @@ func (s *sysInfoCtlTestSuite) TestGetInfo() { func TestControllerSuite(t *testing.T) { suite.Run(t, &sysInfoCtlTestSuite{}) } + +func TestOIDCProviderName(t *testing.T) { + type args struct { + cfg map[string]interface{} + } + tests := []struct { + name string + args args + want string + }{ + {"normal testing", args{map[string]interface{}{common.AUTHMode: common.OIDCAuth, common.OIDCName: "test"}}, "test"}, + {"not oidc", args{map[string]interface{}{common.AUTHMode: common.DBAuth, common.OIDCName: "test"}}, ""}, + {"empty provider", args{map[string]interface{}{common.AUTHMode: common.OIDCAuth, common.OIDCName: ""}}, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := OIDCProviderName(tt.args.cfg); got != tt.want { + t.Errorf("OIDCProviderName() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/src/server/v2.0/handler/systeminfo.go b/src/server/v2.0/handler/systeminfo.go index c2d3435ac26..5d7e46f9529 100644 --- a/src/server/v2.0/handler/systeminfo.go +++ b/src/server/v2.0/handler/systeminfo.go @@ -87,6 +87,7 @@ func (s *sysInfoAPI) convertInfo(d *si.Data) *models.GeneralInfo { SelfRegistration: &d.SelfRegistration, HarborVersion: &d.HarborVersion, BannerMessage: &d.BannerMessage, + OIDCProviderName: &d.OIDCProviderName, } if d.AuthProxySettings != nil { res.AuthproxySettings = &models.AuthproxySetting{ From 6a6e29570cea1ccda47268e1f7f74fc95fd45009 Mon Sep 17 00:00:00 2001 From: Shijun Sun <30999793+AllForNothing@users.noreply.github.com> Date: Wed, 15 Nov 2023 13:23:57 +0800 Subject: [PATCH 02/11] Show OIDC provider name on the OIDC login button (#19581) 1. Related issue #13198 Signed-off-by: AllForNothing --- .../account/sign-in/sign-in.component.html | 22 +++++++++++++++++-- .../account/sign-in/sign-in.component.scss | 4 ++++ .../app/account/sign-in/sign-in.component.ts | 3 +++ src/portal/src/app/services/app-config.ts | 2 ++ src/portal/src/i18n/lang/de-de-lang.json | 1 + src/portal/src/i18n/lang/en-us-lang.json | 3 ++- src/portal/src/i18n/lang/es-es-lang.json | 1 + src/portal/src/i18n/lang/fr-fr-lang.json | 1 + src/portal/src/i18n/lang/pt-br-lang.json | 1 + src/portal/src/i18n/lang/tr-tr-lang.json | 1 + src/portal/src/i18n/lang/zh-cn-lang.json | 1 + src/portal/src/i18n/lang/zh-tw-lang.json | 1 + 12 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/portal/src/app/account/sign-in/sign-in.component.html b/src/portal/src/app/account/sign-in/sign-in.component.html index eeab0c0af70..5108d43164f 100644 --- a/src/portal/src/app/account/sign-in/sign-in.component.html +++ b/src/portal/src/app/account/sign-in/sign-in.component.html @@ -26,8 +26,26 @@
diff --git a/src/portal/src/app/account/sign-in/sign-in.component.scss b/src/portal/src/app/account/sign-in/sign-in.component.scss index 8067e84ade7..acf69b42d28 100644 --- a/src/portal/src/app/account/sign-in/sign-in.component.scss +++ b/src/portal/src/app/account/sign-in/sign-in.component.scss @@ -103,3 +103,7 @@ text-align: right; padding-top: 2rem; } + +.oidc-login-button { + text-transform: unset !important; +} diff --git a/src/portal/src/app/account/sign-in/sign-in.component.ts b/src/portal/src/app/account/sign-in/sign-in.component.ts index 98dbf2b8720..3cb75368e54 100644 --- a/src/portal/src/app/account/sign-in/sign-in.component.ts +++ b/src/portal/src/app/account/sign-in/sign-in.component.ts @@ -151,6 +151,9 @@ export class SignInComponent implements AfterViewChecked, OnInit { CONFIG_AUTH_MODE.OIDC_AUTH ); } + getOIDCProviderName(): string { + return this.appConfigService.getConfig()?.oidc_provider_name || null; + } clickRememberMe($event: any): void { if ($event && $event.target) { this.rememberMe = $event.target.checked; diff --git a/src/portal/src/app/services/app-config.ts b/src/portal/src/app/services/app-config.ts index 0235ffa1a28..5c397ef9830 100644 --- a/src/portal/src/app/services/app-config.ts +++ b/src/portal/src/app/services/app-config.ts @@ -31,6 +31,7 @@ export class AppConfig { show_popular_repo: boolean; banner_message: string; current_time: string; + oidc_provider_name: string; constructor() { // Set default value @@ -52,5 +53,6 @@ export class AppConfig { this.read_only = false; this.show_popular_repo = false; this.banner_message = ''; + this.oidc_provider_name = ''; } } diff --git a/src/portal/src/i18n/lang/de-de-lang.json b/src/portal/src/i18n/lang/de-de-lang.json index 83a02b344a8..90380b50125 100644 --- a/src/portal/src/i18n/lang/de-de-lang.json +++ b/src/portal/src/i18n/lang/de-de-lang.json @@ -28,6 +28,7 @@ "DELETE": "LÖSCHEN", "LOG_IN": "EINLOGGEN", "LOG_IN_OIDC": "Über OIDC Provider einloggen", + "LOG_IN_OIDC_WITH_PROVIDER_NAME": "LOGIN WITH {{providerName}}", "SIGN_UP_LINK": "Neuen Account registrieren", "SIGN_UP": "Registrieren", "CONFIRM": "BESTÄTIGEN", diff --git a/src/portal/src/i18n/lang/en-us-lang.json b/src/portal/src/i18n/lang/en-us-lang.json index bece1ab7673..03142a54cbc 100644 --- a/src/portal/src/i18n/lang/en-us-lang.json +++ b/src/portal/src/i18n/lang/en-us-lang.json @@ -27,7 +27,8 @@ "OK": "OK", "DELETE": "DELETE", "LOG_IN": "LOG IN", - "LOG_IN_OIDC": "Login via OIDC provider", + "LOG_IN_OIDC": "LOGIN VIA OIDC PROVIDER", + "LOG_IN_OIDC_WITH_PROVIDER_NAME": "LOGIN WITH {{providerName}}", "SIGN_UP_LINK": "Sign up for an account", "SIGN_UP": "SIGN UP", "CONFIRM": "CONFIRM", diff --git a/src/portal/src/i18n/lang/es-es-lang.json b/src/portal/src/i18n/lang/es-es-lang.json index 161268d57b8..36261ed5378 100644 --- a/src/portal/src/i18n/lang/es-es-lang.json +++ b/src/portal/src/i18n/lang/es-es-lang.json @@ -28,6 +28,7 @@ "DELETE": "ELIMINAR", "LOG_IN": "IDENTIFICARSE", "LOG_IN_OIDC": "Login Via OIDC proveedor", + "LOG_IN_OIDC_WITH_PROVIDER_NAME": "LOGIN WITH {{providerName}}", "SIGN_UP_LINK": "Registrar una cuenta", "SIGN_UP": "REGISTRARSE", "CONFIRM": "CONFIRMAR", diff --git a/src/portal/src/i18n/lang/fr-fr-lang.json b/src/portal/src/i18n/lang/fr-fr-lang.json index c217779aa36..2caa0710871 100644 --- a/src/portal/src/i18n/lang/fr-fr-lang.json +++ b/src/portal/src/i18n/lang/fr-fr-lang.json @@ -28,6 +28,7 @@ "DELETE": "Supprimer", "LOG_IN": "S'identifier", "LOG_IN_OIDC": "Connexion via fournisseur OIDC", + "LOG_IN_OIDC_WITH_PROVIDER_NAME": "LOGIN WITH {{providerName}}", "SIGN_UP_LINK": "Ouvrir un compte", "SIGN_UP": "S'inscrire", "CONFIRM": "Confirmer", diff --git a/src/portal/src/i18n/lang/pt-br-lang.json b/src/portal/src/i18n/lang/pt-br-lang.json index 98bc450a9ee..af23690352a 100644 --- a/src/portal/src/i18n/lang/pt-br-lang.json +++ b/src/portal/src/i18n/lang/pt-br-lang.json @@ -28,6 +28,7 @@ "DELETE": "REMOVER", "LOG_IN": "ENTRAR", "LOG_IN_OIDC": "Entrar com provedor OIDC", + "LOG_IN_OIDC_WITH_PROVIDER_NAME": "LOGIN WITH {{providerName}}", "SIGN_UP_LINK": "Criar uma nova conta", "SIGN_UP": "CADASTRO", "CONFIRM": "CONFIRMAR", diff --git a/src/portal/src/i18n/lang/tr-tr-lang.json b/src/portal/src/i18n/lang/tr-tr-lang.json index 29cbba0eef6..71fd013dcd5 100644 --- a/src/portal/src/i18n/lang/tr-tr-lang.json +++ b/src/portal/src/i18n/lang/tr-tr-lang.json @@ -28,6 +28,7 @@ "DELETE": "SİL", "LOG_IN": "OTURUM AÇ", "LOG_IN_OIDC": "OIDC sağlayıcı üzerinden giriş yapın", + "LOG_IN_OIDC_WITH_PROVIDER_NAME": "LOGIN WITH {{providerName}}", "SIGN_UP_LINK": "Bir hesap için kayıt olun", "SIGN_UP": "KAYIT OL", "CONFIRM": "ONAYLA", diff --git a/src/portal/src/i18n/lang/zh-cn-lang.json b/src/portal/src/i18n/lang/zh-cn-lang.json index b64ab492697..b808487622f 100644 --- a/src/portal/src/i18n/lang/zh-cn-lang.json +++ b/src/portal/src/i18n/lang/zh-cn-lang.json @@ -28,6 +28,7 @@ "DELETE": "删除", "LOG_IN": "登录", "LOG_IN_OIDC": "通过OIDC提供商登录", + "LOG_IN_OIDC_WITH_PROVIDER_NAME": "通过 {{providerName}} 登录", "SIGN_UP_LINK": "注册账号", "SIGN_UP": "注册", "CONFIRM": "确定", diff --git a/src/portal/src/i18n/lang/zh-tw-lang.json b/src/portal/src/i18n/lang/zh-tw-lang.json index 8a8bef0529a..8618e88e8a4 100644 --- a/src/portal/src/i18n/lang/zh-tw-lang.json +++ b/src/portal/src/i18n/lang/zh-tw-lang.json @@ -28,6 +28,7 @@ "DELETE": "刪除", "LOG_IN": "登入", "LOG_IN_OIDC": "透過 OIDC 提供者登入", + "LOG_IN_OIDC_WITH_PROVIDER_NAME": "LOGIN WITH {{providerName}}", "SIGN_UP_LINK": "註冊帳號", "SIGN_UP": "註冊", "CONFIRM": "確認", From 77181343011df8e745c05de2b5ae35f54980864f Mon Sep 17 00:00:00 2001 From: Shijun Sun <30999793+AllForNothing@users.noreply.github.com> Date: Wed, 15 Nov 2023 15:25:55 +0800 Subject: [PATCH 03/11] Add quota sorting to the project quotas list (#19576) 1. For #16517 Signed-off-by: AllForNothing --- .../project-quotas/project-quotas.component.html | 4 +++- .../project-quotas/project-quotas/project-quotas.component.ts | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/portal/src/app/base/left-side-nav/project-quotas/project-quotas/project-quotas.component.html b/src/portal/src/app/base/left-side-nav/project-quotas/project-quotas/project-quotas.component.html index 51c63c83927..c37510b7c06 100644 --- a/src/portal/src/app/base/left-side-nav/project-quotas/project-quotas/project-quotas.component.html +++ b/src/portal/src/app/base/left-side-nav/project-quotas/project-quotas/project-quotas.component.html @@ -67,7 +67,9 @@ {{ 'QUOTA.PROJECT' | translate }} {{ 'QUOTA.OWNER' | translate }} - {{ 'QUOTA.STORAGE' | translate }} + {{ + 'QUOTA.STORAGE' | translate + }} {{ 'QUOTA.PLACEHOLDER' | translate }} diff --git a/src/portal/src/app/base/left-side-nav/project-quotas/project-quotas/project-quotas.component.ts b/src/portal/src/app/base/left-side-nav/project-quotas/project-quotas/project-quotas.component.ts index 302dc153489..16aab76af89 100644 --- a/src/portal/src/app/base/left-side-nav/project-quotas/project-quotas/project-quotas.component.ts +++ b/src/portal/src/app/base/left-side-nav/project-quotas/project-quotas/project-quotas.component.ts @@ -20,6 +20,7 @@ import { getPageSizeFromLocalStorage, PageSizeMapKeys, setPageSizeToLocalStorage, + getSortingString, } from '../../../../shared/units/utils'; import { ErrorHandler } from '../../../../shared/units/error-handler'; import { @@ -242,6 +243,7 @@ export class ProjectQuotasComponent implements OnChanges { reference: QuotaType, page: pageNumber, pageSize: this.pageSize, + sort: getSortingString(state), }) .pipe( finalize(() => { From 30730c6716e4c3640a5654421146181c9e8c520b Mon Sep 17 00:00:00 2001 From: Shijun Sun <30999793+AllForNothing@users.noreply.github.com> Date: Wed, 15 Nov 2023 16:23:07 +0800 Subject: [PATCH 04/11] Add a placeholder to the cards for the security-hub (#19536) 1. Related issue #19249 Signed-off-by: AllForNothing --- .../vulnerability-summary.component.html | 226 ++++++++++-------- .../vulnerability-summary.component.scss | 22 ++ .../vulnerability-summary.component.ts | 4 + 3 files changed, 157 insertions(+), 95 deletions(-) diff --git a/src/portal/src/app/base/left-side-nav/interrogation-services/vulnerability-database/vulnerability-summary/vulnerability-summary.component.html b/src/portal/src/app/base/left-side-nav/interrogation-services/vulnerability-database/vulnerability-summary/vulnerability-summary.component.html index 8f72132e4de..7d4789f5a6d 100644 --- a/src/portal/src/app/base/left-side-nav/interrogation-services/vulnerability-database/vulnerability-summary/vulnerability-summary.component.html +++ b/src/portal/src/app/base/left-side-nav/interrogation-services/vulnerability-database/vulnerability-summary/vulnerability-summary.component.html @@ -119,43 +119,55 @@

-
-
- - - {{ item.repository_name }} - - -
-
- {{ - item?.digest?.slice(0, 15) - }} - -
-
-
- +
+
+ + + {{ item.repository_name }} + +
-
+
+ {{ + item?.digest?.slice(0, 15) + }} + +
+
+
+ +
+
+
+
+ {{ + 'SECURITY_HUB.NO_VUL' | translate + }}
@@ -178,67 +190,91 @@

-
-
- - - {{ item.cve_id }} - - -
-
- - {{ severityText(item.severity) | translate }} - {{ severityText(item.severity) | translate }} - {{ severityText(item.severity) | translate }} +
+
+ + + {{ item.cve_id }} + + +
+
+ + {{ + severityText(item.severity) | translate + }} + {{ + severityText(item.severity) | translate + }} + {{ + severityText(item.severity) | translate + }} + {{ + severityText(item.severity) | translate + }} + {{ + severityText(item.severity) | translate + }} + {{ + severityText(item.severity) | translate + }} + +
+
+ {{ item?.cvss_score_v3 }} +
+
{{ severityText(item.severity) | translate }} + *ngIf="item?.version" + title="{{ item?.package + '@' + item?.version }}"> + {{ item?.package + '@' + item?.version }} + {{ severityText(item.severity) | translate }} - {{ - severityText(item.severity) | translate - }} - -
-
- {{ item?.cvss_score_v3 }} -
-
- - {{ item?.package + '@' + item?.version }} - - - {{ item?.package }} - -
+ *ngIf="!item?.version" + title="{{ item?.package }}"> + {{ item?.package }} + +
+
+
+
+ {{ + 'SECURITY_HUB.NO_VUL' | translate + }}
diff --git a/src/portal/src/app/base/left-side-nav/interrogation-services/vulnerability-database/vulnerability-summary/vulnerability-summary.component.scss b/src/portal/src/app/base/left-side-nav/interrogation-services/vulnerability-database/vulnerability-summary/vulnerability-summary.component.scss index 1641295b787..e3bde20c6f5 100644 --- a/src/portal/src/app/base/left-side-nav/interrogation-services/vulnerability-database/vulnerability-summary/vulnerability-summary.component.scss +++ b/src/portal/src/app/base/left-side-nav/interrogation-services/vulnerability-database/vulnerability-summary/vulnerability-summary.component.scss @@ -110,3 +110,25 @@ $row-height: 48px; .label { min-width: 3rem; } + +.datagrid-placeholder { + margin-top: 2.5rem; + border-top: 0; + padding: 0.6rem; + display: flex; + flex-flow: column nowrap; + align-items: center; + justify-content: flex-start; + font-size: .8rem; + color: #b3b3b3; +} + +.datagrid-placeholder-image { + height: 3rem; + width: 3rem; + margin-bottom: 0.6rem; + background-repeat: no-repeat; + background-size: contain; + background-position: center; + background-image: url("data:image/svg+xml;charset=utf8,%3Csvg%20viewBox%3D%220%200%2060%2072%22%20version%3D%221.1%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20xmlns%3Axlink%3D%22http%3A%2F%2Fwww.w3.org%2F1999%2Fxlink%22%3E%0A%20%20%20%20%3Cdefs%3E%0A%20%20%20%20%20%20%20%20%3Cellipse%20id%3D%22path-1%22%20cx%3D%2230%22%20cy%3D%2261.7666667%22%20rx%3D%2215.4512904%22%20ry%3D%224.73333333%22%3E%3C%2Fellipse%3E%0A%20%20%20%20%20%20%20%20%3Cmask%20id%3D%22mask-2%22%20maskContentUnits%3D%22userSpaceOnUse%22%20maskUnits%3D%22objectBoundingBox%22%20x%3D%220%22%20y%3D%220%22%20width%3D%2230.9025808%22%20height%3D%229.46666667%22%20fill%3D%22white%22%3E%0A%20%20%20%20%20%20%20%20%20%20%20%20%3Cuse%20xlink%3Ahref%3D%22%23path-1%22%3E%3C%2Fuse%3E%0A%20%20%20%20%20%20%20%20%3C%2Fmask%3E%0A%20%20%20%20%3C%2Fdefs%3E%0A%20%20%20%20%3Cg%20id%3D%22Page-1%22%20stroke%3D%22none%22%20stroke-width%3D%221%22%20fill%3D%22none%22%20fill-rule%3D%22evenodd%22%3E%0A%20%20%20%20%20%20%20%20%3Cg%20id%3D%22Artboard%22%3E%0A%20%20%20%20%20%20%20%20%20%20%20%20%3Cuse%20id%3D%22Oval-10%22%20stroke%3D%22%23C1DFEF%22%20mask%3D%22url(%23mask-2)%22%20stroke-width%3D%222.8%22%20stroke-linecap%3D%22square%22%20stroke-dasharray%3D%223%2C6%2C3%2C5%22%20xlink%3Ahref%3D%22%23path-1%22%3E%3C%2Fuse%3E%0A%20%20%20%20%20%20%20%20%20%20%20%20%3Cpath%20d%3D%22M38.4613647%2C18.1642456%20L30.9890137%2C34.9141846%20L31%2C47%20L32.5977783%2C46.5167236%20L32.5977783%2C34.9141846%20L51.0673218%2C15.7560425%20C51.0673218%2C15.7560425%2048.6295166%2C16.6542969%2044.9628906%2C17.3392334%20C41.2962646%2C18.0241699%2038.4613647%2C18.1642456%2038.4613647%2C18.1642456%20Z%22%20id%3D%22Path-195%22%20fill%3D%22%23C1DFEF%22%3E%3C%2Fpath%3E%0A%20%20%20%20%20%20%20%20%20%20%20%20%3Cpath%20d%3D%22M4.74639226%2C12.5661855%20L4.62065726%2C12.1605348%20L5.3515414%2C11.1625044%20L5.77622385%2C11.159939%20L6.20936309%2C12.5573481%20L4.74639226%2C12.5661855%20Z%20M6.20936309%2C12.5573481%20L6.32542632%2C12.9317954%20L28.4963855%2C34.8796718%20L28.4963855%2C47.8096691%20L32.6%2C46.4836513%20L32.6%2C34.8992365%20L53.973494%2C12.7035813%20L53.973494%2C12.2688201%20L6.20936309%2C12.5573481%20Z%20M55.373494%2C10.8603376%20L55.373494%2C13.2680664%20L34%2C35.4637216%20L34%2C47.5025401%20L27.0963855%2C49.7333333%20L27.0963855%2C35.4637219%20L5.09179688%2C13.680542%20L4.31325301%2C11.1687764%20L55.373494%2C10.8603376%20Z%22%20id%3D%22Path-149%22%20fill%3D%22%237FBDDD%22%3E%3C%2Fpath%3E%0A%20%20%20%20%20%20%20%20%20%20%20%20%3Cellipse%20id%3D%22Oval-9%22%20fill%3D%22%23FFFFFF%22%20cx%3D%2230%22%20cy%3D%2211.785654%22%20rx%3D%2226%22%20ry%3D%226.78565401%22%3E%3C%2Fellipse%3E%0A%20%20%20%20%20%20%20%20%20%20%20%20%3Cpath%20d%3D%22M30%2C17.171308%20C36.8772177%2C17.171308%2043.3112282%2C16.4610701%2048.0312371%2C15.2292106%20C50.2777611%2C14.6428977%2052.0507619%2C13.9579677%2053.2216231%2C13.2354973%20C54.1938565%2C12.6355886%2054.6%2C12.1175891%2054.6%2C11.785654%20C54.6%2C11.4537189%2054.1938565%2C10.9357194%2053.2216231%2C10.3358107%20C52.0507619%2C9.61334032%2050.2777611%2C8.92841034%2048.0312371%2C8.34209746%20C43.3112282%2C7.11023795%2036.8772177%2C6.4%2030%2C6.4%20C23.1227823%2C6.4%2016.6887718%2C7.11023795%2011.9687629%2C8.34209746%20C9.72223886%2C8.92841034%207.94923814%2C9.61334032%206.77837689%2C10.3358107%20C5.8061435%2C10.9357194%205.4%2C11.4537189%205.4%2C11.785654%20C5.4%2C12.1175891%205.8061435%2C12.6355886%206.77837689%2C13.2354973%20C7.94923814%2C13.9579677%209.72223886%2C14.6428977%2011.9687629%2C15.2292106%20C16.6887718%2C16.4610701%2023.1227823%2C17.171308%2030%2C17.171308%20Z%20M30%2C18.571308%20C15.6405965%2C18.571308%204%2C15.5332672%204%2C11.785654%20C4%2C8.03804078%2015.6405965%2C5%2030%2C5%20C44.3594035%2C5%2056%2C8.03804078%2056%2C11.785654%20C56%2C15.5332672%2044.3594035%2C18.571308%2030%2C18.571308%20Z%22%20id%3D%22Oval-9-Copy%22%20fill%3D%22%237FBDDD%22%3E%3C%2Fpath%3E%0A%20%20%20%20%20%20%20%20%20%20%20%20%3Cpath%20d%3D%22M18.2608643%2C7.14562988%20L22.727356%2C16.9047241%20C22.727356%2C16.9047241%2015.3006592%2C16.3911743%2010.276001%2C14.7511597%20C5.25134277%2C13.111145%205.38031006%2C11.8284302%205.38031006%2C11.6882935%20C5.38031006%2C10.4832831%208.16633152%2C9.41877716%2011.114563%2C8.57324219%20C14.549319%2C7.58817492%2018.2608643%2C7.14562988%2018.2608643%2C7.14562988%20Z%22%20id%3D%22Path-196%22%20fill%3D%22%23C1DFEF%22%3E%3C%2Fpath%3E%0A%20%20%20%20%20%20%20%20%3C%2Fg%3E%0A%20%20%20%20%3C%2Fg%3E%0A%3C%2Fsvg%3E"); +} diff --git a/src/portal/src/app/base/left-side-nav/interrogation-services/vulnerability-database/vulnerability-summary/vulnerability-summary.component.ts b/src/portal/src/app/base/left-side-nav/interrogation-services/vulnerability-database/vulnerability-summary/vulnerability-summary.component.ts index 19b3b947f11..5f779a7dcc2 100644 --- a/src/portal/src/app/base/left-side-nav/interrogation-services/vulnerability-database/vulnerability-summary/vulnerability-summary.component.ts +++ b/src/portal/src/app/base/left-side-nav/interrogation-services/vulnerability-database/vulnerability-summary/vulnerability-summary.component.ts @@ -26,6 +26,7 @@ import { import { TranslateService } from '@ngx-translate/core'; import { DangerousArtifact } from '../../../../../../../ng-swagger-gen/models/dangerous-artifact'; import * as echarts from 'echarts/core'; +import { finalize } from 'rxjs/operators'; @Component({ selector: 'app-vulnerability-summary', @@ -45,6 +46,7 @@ export class VulnerabilitySummaryComponent implements OnInit, OnDestroy { chart: any; @ViewChild('pieChart', { static: true }) pieChartEle: ElementRef; + loadingSummary: boolean = false; constructor( private securityHubService: SecurityhubService, private messageHandler: MessageHandlerService, @@ -74,11 +76,13 @@ export class VulnerabilitySummaryComponent implements OnInit, OnDestroy { } getSummary() { + this.loadingSummary = true; this.securityHubService .getSecuritySummary({ withDangerousArtifact: true, withDangerousCve: true, }) + .pipe(finalize(() => (this.loadingSummary = false))) .subscribe({ next: res => { this.securitySummary = res; From da9e263ca7a99ff20dc0289fc69efe4c20e6e6e9 Mon Sep 17 00:00:00 2001 From: Shijun Sun <30999793+AllForNothing@users.noreply.github.com> Date: Wed, 15 Nov 2023 17:12:55 +0800 Subject: [PATCH 05/11] Avoid menu closure when filtering labels (#19561) 1. Fixes #19554 Signed-off-by: AllForNothing --- .../artifact-list-tab.component.html | 3 +- .../label-selector.component.html | 54 +++++++++++++------ .../label-selector.component.ts | 2 + 3 files changed, 43 insertions(+), 16 deletions(-) diff --git a/src/portal/src/app/base/project/repository/artifact/artifact-list-page/artifact-list/artifact-list-tab/artifact-list-tab.component.html b/src/portal/src/app/base/project/repository/artifact/artifact-list-page/artifact-list/artifact-list-tab/artifact-list-tab.component.html index 33117a0532a..6258f7a7d26 100644 --- a/src/portal/src/app/base/project/repository/artifact/artifact-list-page/artifact-list/artifact-list-tab/artifact-list-tab.component.html +++ b/src/portal/src/app/base/project/repository/artifact/artifact-list-page/artifact-list/artifact-list-tab/artifact-list-tab.component.html @@ -80,12 +80,13 @@ -
+
{{ 'LABEL.NO_LABELS' | translate }}
-
- -
+ +
+ +
+ + +
+ +
+
diff --git a/src/portal/src/app/shared/components/label-selector/label-selector.component.ts b/src/portal/src/app/shared/components/label-selector/label-selector.component.ts index 54264afb9b7..6a91207997e 100644 --- a/src/portal/src/app/shared/components/label-selector/label-selector.component.ts +++ b/src/portal/src/app/shared/components/label-selector/label-selector.component.ts @@ -31,6 +31,8 @@ const PAGE_SIZE: number = 50; styleUrls: ['./label-selector.component.scss'], }) export class LabelSelectorComponent implements OnInit, OnChanges, OnDestroy { + @Input() + usedInDropdown: boolean = false; @Input() ownedLabels: Label[] = []; @Input() From 95b40322010f57b623a2f1c0299914fff6565a31 Mon Sep 17 00:00:00 2001 From: Shijun Sun <30999793+AllForNothing@users.noreply.github.com> Date: Wed, 15 Nov 2023 17:50:06 +0800 Subject: [PATCH 06/11] Correct loop condition for replication tasks (#19570) Signed-off-by: AllForNothing --- src/portal/PACKAGE.md | 2 +- .../replication-tasks/replication-tasks.component.ts | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/portal/PACKAGE.md b/src/portal/PACKAGE.md index e8e0a627931..69471aa40b9 100644 --- a/src/portal/PACKAGE.md +++ b/src/portal/PACKAGE.md @@ -1,7 +1,7 @@ ```text { "name": "harbor", - "version": "2.9.0", + "version": "2.10.0", "description": "Harbor UI with Clarity", "angular-cli": {}, "scripts": { diff --git a/src/portal/src/app/base/left-side-nav/replication/replication/replication-tasks/replication-tasks.component.ts b/src/portal/src/app/base/left-side-nav/replication/replication/replication-tasks/replication-tasks.component.ts index 1162ab98c4b..c3a39b3023a 100644 --- a/src/portal/src/app/base/left-side-nav/replication/replication/replication-tasks/replication-tasks.component.ts +++ b/src/portal/src/app/base/left-side-nav/replication/replication/replication-tasks/replication-tasks.component.ts @@ -252,13 +252,11 @@ export class ReplicationTasksComponent implements OnInit, OnDestroy { count > 0 || this.execution?.status === executionStatus ) { - if (!this.tasksTimeout) { - this.tasksTimeout = setTimeout(() => { - this.clrLoadTasks(false, { - page: {}, - }); - }, REFRESH_TIME_DIFFERENCE); - } + this.tasksTimeout = setTimeout(() => { + this.clrLoadTasks(false, { + page: {}, + }); + }, REFRESH_TIME_DIFFERENCE); } }, error => { From 08cb0803a8978a5f1bf17d984420e5fb484d61b4 Mon Sep 17 00:00:00 2001 From: Yang Jiao <72076317+YangJiao0817@users.noreply.github.com> Date: Thu, 16 Nov 2023 16:51:00 +0800 Subject: [PATCH 07/11] Add API(audit-log,project,registry,replication) permission testcase (#19584) Add audit-log,project,registry,replication API permission testcase Signed-off-by: Yang Jiao --- tests/apitests/python/test_permission.py | 197 +++++++++++++++++++++-- 1 file changed, 186 insertions(+), 11 deletions(-) diff --git a/tests/apitests/python/test_permission.py b/tests/apitests/python/test_permission.py index ee123c8eba9..ee9f4d20edd 100644 --- a/tests/apitests/python/test_permission.py +++ b/tests/apitests/python/test_permission.py @@ -3,36 +3,47 @@ import requests import urllib3 import os +from urllib.parse import urlsplit user_name = os.environ.get("USER_NAME") password = os.environ.get("PASSWORD") +admin_user_name = os.environ.get("ADMIN_USER_NAME") +admin_password = os.environ.get("ADMIN_PASSWORD") harbor_base_url = os.environ.get("HARBOR_BASE_URL") resource = os.environ.get("RESOURCE") - +ID_PLACEHOLDER = "(id)" +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) class Permission: - def __init__(self, url, method, expect_status_code, payload=None, res_id_field=None, payload_id_field=None): + def __init__(self, url, method, expect_status_code, payload=None, res_id_field=None, payload_id_field=None, id_from_header=False): self.url = url self.method = method self.expect_status_code = expect_status_code self.payload = payload self.res_id_field = res_id_field self.payload_id_field = payload_id_field if payload_id_field else res_id_field + self.id_from_header = id_from_header def call(self): - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - response = None - requests.get + if ID_PLACEHOLDER in self.url: + self.url = self.url.replace(ID_PLACEHOLDER, str(self.payload.get(self.payload_id_field))) response = requests.request(self.method, self.url, data=json.dumps(self.payload), verify=False, auth=(user_name, password), headers={"Content-Type": "application/json"}) - print(response.text) assert response.status_code == self.expect_status_code, "Failed to call the {} {}, expected status code is {}, but got {}, error msg is {}".format(self.method, self.url, self.expect_status_code, response.status_code, response.text) - if self.res_id_field and self.payload_id_field: - self.payload[self.payload_id_field] = json.loads(response.text)[self.res_id_field] + if self.res_id_field and self.payload_id_field and self.id_from_header == False: + self.payload[self.payload_id_field] = int(json.loads(response.text)[self.res_id_field]) + elif self.res_id_field and self.payload_id_field and self.id_from_header == True: + self.payload[self.payload_id_field] = int(response.headers["Location"].split("/")[-1]) +resource_permissions = {} +# audit logs permissions start +list_audit_logs = Permission("{}/audit-logs".format(harbor_base_url), "GET", 200) +audit_log = [ list_audit_logs ] +resource_permissions["audit-log"] = audit_log +# audit logs permissions end # preheat instance permissions start preheat_instance_payload = { @@ -50,14 +61,178 @@ def call(self): delete_preheat_instance = Permission("{}/p2p/preheat/instances/{}".format(harbor_base_url, preheat_instance_payload["name"]), "DELETE", 200, preheat_instance_payload) ping_preheat_instance = Permission("{}/p2p/preheat/instances/ping".format(harbor_base_url), "POST", 500, preheat_instance_payload) preheat_instances = [ create_preheat_instance, list_preheat_instance, read_preheat_instance, update_preheat_instance, delete_preheat_instance ] -resource_permissions = { - "preheat-instance": preheat_instances -} +resource_permissions["preheat-instance"] = preheat_instances # preheat instance permissions end +# project permissions start +project_payload = { + "metadata": { + "public": "false" + }, + "project_name": "project-{}".format(random.randint(1000, 9999)), + "storage_limit": -1 +} +create_project = Permission("{}/projects".format(harbor_base_url), "POST", 201, project_payload) +list_project = Permission("{}/projects".format(harbor_base_url), "GET", 200, project_payload) +project = [ create_project, list_project ] +resource_permissions["project"] = project +# project permissions end + +# registry permissions start +registry_payload = { + "insecure": False, + "name": "registry-{}".format(random.randint(1000, 9999)), + "type": "docker-hub", + "url": "https://hub.docker.com" +} +create_registry = Permission("{}/registries".format(harbor_base_url), "POST", 201, registry_payload, "id", id_from_header=True) +list_registry = Permission("{}/registries".format(harbor_base_url), "GET", 200, registry_payload) +read_registry = Permission("{}/registries/{}".format(harbor_base_url, ID_PLACEHOLDER), "GET", 200, registry_payload, payload_id_field="id") +info_registry = Permission("{}/registries/{}/info".format(harbor_base_url, ID_PLACEHOLDER), "GET", 200, registry_payload, payload_id_field="id") +update_registry = Permission("{}/registries/{}".format(harbor_base_url, ID_PLACEHOLDER), "PUT", 200, registry_payload, payload_id_field="id") +delete_registry = Permission("{}/registries/{}".format(harbor_base_url, ID_PLACEHOLDER), "DELETE", 200, registry_payload, payload_id_field="id") +registry_ping_payload = { + "insecure": False, + "name": "registry-{}".format(random.randint(1000, 9999)), + "type": "docker-hub", + "url": "https://hub.docker.com" +} +ping_registry = Permission("{}/registries/ping".format(harbor_base_url), "POST", 200, registry_ping_payload) +registry = [ create_registry, list_registry, read_registry, info_registry, update_registry, delete_registry, ping_registry ] +resource_permissions["registry"] = registry +# registry permissions end + +# replication-adapter permissions start +list_replication_adapters = Permission("{}/replication/adapters".format(harbor_base_url), "GET", 200) +list_replication_adapterinfos = Permission("{}/replication/adapterinfos".format(harbor_base_url), "GET", 200) +replication_adapter = [ list_replication_adapters, list_replication_adapterinfos ] +resource_permissions["replication-adapter"] = replication_adapter +# replication-adapter permissions end + +# replication policy permissions start +replication_registry_id = None +replication_registry_name = "replication-registry-{}".format(random.randint(1000, 9999)) +if resource == "replication-policy": + result = urlsplit(harbor_base_url) + endpoint_URL = "{}://{}".format(result.scheme, result.netloc) + replication_registry_payload = { + "credential": { + "access_key": admin_user_name, + "access_secret": admin_password, + "type": "basic" + }, + "description": "", + "insecure": True, + "name": replication_registry_name, + "type": "harbor", + "url": endpoint_URL + } + response = requests.post("{}/registries".format(harbor_base_url), data=json.dumps(replication_registry_payload), verify=False, auth=(admin_user_name, admin_password), headers={"Content-Type": "application/json"}) + replication_registry_id = int(response.headers["Location"].split("/")[-1]) +replication_policy_payload = { + "name": "replication_policy_{}".format(random.randint(1000, 9999)), + "src_registry": None, + "dest_registry": { + "id": replication_registry_id + }, + "dest_namespace": "library", + "dest_namespace_replace_count": 1, + "trigger": { + "type": "manual", + "trigger_settings": { + "cron": "" + } + }, + "filters": [ + { + "type": "name", + "value": "library/**" + } + ], + "enabled": True, + "deletion": False, + "override": True, + "speed": -1, + "copy_by_chunk": False +} +create_replication_policy = Permission("{}/replication/policies".format(harbor_base_url), "POST", 201, replication_policy_payload, "id", id_from_header=True) +list_replication_policy = Permission("{}/replication/policies".format(harbor_base_url), "GET", 200, replication_policy_payload) +read_replication_policy = Permission("{}/replication/policies/{}".format(harbor_base_url, ID_PLACEHOLDER), "GET", 200, replication_policy_payload, payload_id_field="id") +update_replication_policy = Permission("{}/replication/policies/{}".format(harbor_base_url, ID_PLACEHOLDER), "PUT", 200, replication_policy_payload, payload_id_field="id") +delete_replication_policy = Permission("{}/replication/policies/{}".format(harbor_base_url, ID_PLACEHOLDER), "DELETE", 200, replication_policy_payload, payload_id_field="id") +replication_and_policy = [ create_replication_policy, list_replication_policy, read_replication_policy, update_replication_policy, delete_replication_policy ] +resource_permissions["replication-policy"] = replication_and_policy +# replication policy permissions end + +# replication permissions start +replication_policy_id = None +replication_policy_name = "replication-policy-{}".format(random.randint(1000, 9999)) +if resource == "replication": + result = urlsplit(harbor_base_url) + endpoint_URL = "{}://{}".format(result.scheme, result.netloc) + replication_registry_payload = { + "credential": { + "access_key": admin_user_name, + "access_secret": admin_password, + "type": "basic" + }, + "description": "", + "insecure": True, + "name": "replication-registry-{}".format(random.randint(1000, 9999)), + "type": "harbor", + "url": endpoint_URL + } + response = requests.post("{}/registries".format(harbor_base_url), data=json.dumps(replication_registry_payload), verify=False, auth=(admin_user_name, admin_password), headers={"Content-Type": "application/json"}) + replication_registry_id = int(response.headers["Location"].split("/")[-1]) + replication_policy_payload = { + "name": replication_policy_name, + "src_registry": None, + "dest_registry": { + "id": replication_registry_id + }, + "dest_namespace": "library", + "dest_namespace_replace_count": 1, + "trigger": { + "type": "manual", + "trigger_settings": { + "cron": "" + } + }, + "filters": [ + { + "type": "name", + "value": "library/**" + } + ], + "enabled": True, + "deletion": False, + "override": True, + "speed": -1, + "copy_by_chunk": False + } + response = requests.post("{}/replication/policies".format(harbor_base_url), data=json.dumps(replication_policy_payload), verify=False, auth=(admin_user_name, admin_password), headers={"Content-Type": "application/json"}) + replication_policy_id = int(response.headers["Location"].split("/")[-1]) +replication_execution_payload = { + "policy_id": replication_policy_id +} +create_replication_execution = Permission("{}/replication/executions".format(harbor_base_url), "POST", 201, replication_execution_payload, "id", id_from_header=True) +list_replication_execution = Permission("{}/replication/executions".format(harbor_base_url), "GET", 200, replication_execution_payload) +read_replication_execution = Permission("{}/replication/executions/{}".format(harbor_base_url, ID_PLACEHOLDER), "GET", 200, replication_execution_payload, payload_id_field="id") +stop_replication_execution = Permission("{}/replication/executions/{}".format(harbor_base_url, ID_PLACEHOLDER), "PUT", 200, replication_execution_payload, payload_id_field="id") +list_replication_execution_tasks = Permission("{}/replication/executions/{}/tasks".format(harbor_base_url, ID_PLACEHOLDER), "GET", 200, replication_execution_payload, payload_id_field="id") +read_replication_execution_task = Permission("{}/replication/executions/{}/tasks/{}".format(harbor_base_url, ID_PLACEHOLDER, 1), "GET", 404, replication_execution_payload, payload_id_field="id") +replication = [ create_replication_execution, list_replication_execution, read_replication_execution, stop_replication_execution, list_replication_execution_tasks, read_replication_execution_task ] +resource_permissions["replication"] = replication +# replication permissions end + + def main(): for permission in resource_permissions[resource]: + print("=================================================") + print("call: {} {}".format(permission.method, permission.url)) + print("payload: {}".format(json.dumps(permission.payload))) + print("=================================================\n") permission.call() From 307a3121aa4646fb08d77974f526f70a929ca047 Mon Sep 17 00:00:00 2001 From: "stonezdj(Daojun Zhang)" Date: Fri, 17 Nov 2023 15:29:19 +0800 Subject: [PATCH 08/11] Return empty result when no scanner configured (#19577) fixes #19534 Signed-off-by: stonezdj --- src/controller/securityhub/controller.go | 4 ++-- src/pkg/scan/scanner/manager.go | 3 +++ src/server/v2.0/handler/security.go | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/controller/securityhub/controller.go b/src/controller/securityhub/controller.go index f53527a5ddb..80babcb9ab3 100644 --- a/src/controller/securityhub/controller.go +++ b/src/controller/securityhub/controller.go @@ -86,8 +86,8 @@ func NewController() Controller { func (c *controller) SecuritySummary(ctx context.Context, projectID int64, options ...Option) (*secHubModel.Summary, error) { opts := newOptions(options...) scannerUUID, err := c.scannerMgr.DefaultScannerUUID(ctx) - if err != nil { - return nil, err + if len(scannerUUID) == 0 || err != nil { + return &secHubModel.Summary{}, nil } sum, err := c.secHubMgr.Summary(ctx, scannerUUID, projectID, nil) if err != nil { diff --git a/src/pkg/scan/scanner/manager.go b/src/pkg/scan/scanner/manager.go index 96598f20b1c..ed73e38ace6 100644 --- a/src/pkg/scan/scanner/manager.go +++ b/src/pkg/scan/scanner/manager.go @@ -152,5 +152,8 @@ func (bm *basicManager) DefaultScannerUUID(ctx context.Context) (string, error) if err != nil { return "", err } + if reg == nil { + return "", nil + } return reg.UUID, nil } diff --git a/src/server/v2.0/handler/security.go b/src/server/v2.0/handler/security.go index 27aba998fcf..d324b97e5de 100644 --- a/src/server/v2.0/handler/security.go +++ b/src/server/v2.0/handler/security.go @@ -108,8 +108,8 @@ func (s *securityAPI) ListVulnerabilities(ctx context.Context, params securityMo return s.SendError(ctx, err) } scannerUUID, err := scanner.Mgr.DefaultScannerUUID(ctx) - if err != nil { - return s.SendError(ctx, err) + if err != nil || len(scannerUUID) == 0 { + return securityModel.NewListVulnerabilitiesOK().WithPayload([]*models.VulnerabilityItem{}).WithXTotalCount(0) } cnt, err := s.controller.CountVuls(ctx, scannerUUID, 0, *params.TuneCount, query) if err != nil { From 996e57b511f52e42571e8f549bde013973fc8da8 Mon Sep 17 00:00:00 2001 From: Maksym Trofimenko Date: Mon, 20 Nov 2023 20:31:22 +0000 Subject: [PATCH 09/11] Feature: GDPR compliant audit logs (#17396) --- src/common/const.go | 1 + src/controller/user/controller.go | 37 +++++++- .../job/impl/gdpr/audit_logs_data_masking.go | 85 +++++++++++++++++++ .../impl/gdpr/audit_logs_data_masking_test.go | 67 +++++++++++++++ src/jobservice/job/known_jobs.go | 2 + src/jobservice/runtime/bootstrap.go | 13 +-- src/lib/config/metadata/metadatalist.go | 1 + src/lib/config/models/model.go | 1 + src/lib/config/userconfig.go | 1 + src/pkg/audit/dao/dao.go | 11 +++ src/pkg/audit/manager.go | 6 ++ src/pkg/user/manager.go | 17 ++-- src/pkg/user/manager_test.go | 6 +- src/testing/pkg/audit/dao/dao.go | 14 +++ src/testing/pkg/audit/manager.go | 14 +++ src/testing/pkg/user/manager.go | 14 +++ .../python/test_job_service_dashboard.py | 2 +- 17 files changed, 274 insertions(+), 18 deletions(-) create mode 100644 src/jobservice/job/impl/gdpr/audit_logs_data_masking.go create mode 100644 src/jobservice/job/impl/gdpr/audit_logs_data_masking_test.go diff --git a/src/common/const.go b/src/common/const.go index 2daf68299c5..a00601fcc40 100644 --- a/src/common/const.go +++ b/src/common/const.go @@ -184,6 +184,7 @@ const ( TraceOtelTimeout = "trace_otel_timeout" GDPRDeleteUser = "gdpr_delete_user" + GDPRAuditLogs = "gdpr_audit_logs" // These variables are temporary solution for issue: https://github.com/goharbor/harbor/issues/16039 // When user disable the pull count/time/audit log, it will decrease the database access, especially in large concurrency pull scenarios. diff --git a/src/controller/user/controller.go b/src/controller/user/controller.go index 5025ff6af22..1d0901206b4 100644 --- a/src/controller/user/controller.go +++ b/src/controller/user/controller.go @@ -21,12 +21,15 @@ import ( commonmodels "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/security" "github.com/goharbor/harbor/src/common/security/local" + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/jobservice/job/impl/gdpr" "github.com/goharbor/harbor/src/lib" "github.com/goharbor/harbor/src/lib/config" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/pkg/member" "github.com/goharbor/harbor/src/pkg/oidc" + "github.com/goharbor/harbor/src/pkg/task" "github.com/goharbor/harbor/src/pkg/user" "github.com/goharbor/harbor/src/pkg/user/models" ) @@ -76,6 +79,8 @@ func NewController() Controller { mgr: user.New(), oidcMetaMgr: oidc.NewMetaMgr(), memberMgr: member.Mgr, + taskMgr: task.NewManager(), + exeMgr: task.NewExecutionManager(), } } @@ -88,6 +93,8 @@ type controller struct { mgr user.Manager oidcMetaMgr oidc.MetaManager memberMgr member.Manager + taskMgr task.Manager + exeMgr task.ExecutionManager } func (c *controller) UpdateOIDCMeta(ctx context.Context, ou *commonmodels.OIDCUser, cols ...string) error { @@ -183,10 +190,36 @@ func (c *controller) Delete(ctx context.Context, id int) error { if err != nil { return errors.UnknownError(err).WithMessage("failed to load GDPR setting: %v", err) } + + if gdprSetting.AuditLogs { + userDb, err := c.mgr.Get(ctx, id) + if err != nil { + return errors.Wrap(err, "unable to get user information") + } + params := map[string]interface{}{ + gdpr.UserNameParam: userDb.Username, + } + execID, err := c.exeMgr.Create(ctx, job.AuditLogsGDPRCompliantVendorType, -1, task.ExecutionTriggerEvent, params) + if err != nil { + return err + } + _, err = c.taskMgr.Create(ctx, execID, &task.Job{ + Name: job.AuditLogsGDPRCompliantVendorType, + Metadata: &job.Metadata{ + JobKind: job.KindGeneric, + }, + Parameters: params, + }) + if err != nil { + return err + } + } if gdprSetting.DeleteUser { - return c.mgr.DeleteGDPR(ctx, id) + err = c.mgr.DeleteGDPR(ctx, id) + } else { + err = c.mgr.Delete(ctx, id) } - return c.mgr.Delete(ctx, id) + return err } func (c *controller) List(ctx context.Context, query *q.Query, options ...models.Option) ([]*commonmodels.User, error) { diff --git a/src/jobservice/job/impl/gdpr/audit_logs_data_masking.go b/src/jobservice/job/impl/gdpr/audit_logs_data_masking.go new file mode 100644 index 00000000000..2d6832e60c3 --- /dev/null +++ b/src/jobservice/job/impl/gdpr/audit_logs_data_masking.go @@ -0,0 +1,85 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gdpr + +import ( + "fmt" + + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/pkg/audit" + "github.com/goharbor/harbor/src/pkg/user" +) + +const UserNameParam = "username" + +type AuditLogsDataMasking struct { + manager audit.Manager + userManager user.Manager +} + +func (a AuditLogsDataMasking) MaxFails() uint { + return 3 +} + +func (a AuditLogsDataMasking) MaxCurrency() uint { + return 1 +} + +func (a AuditLogsDataMasking) ShouldRetry() bool { + return true +} + +func (a AuditLogsDataMasking) Validate(params job.Parameters) error { + if params == nil { + // Params are required + return errors.New("missing job parameters") + } + _, err := a.parseParams(params) + return err +} + +func (a *AuditLogsDataMasking) init() { + if a.manager == nil { + a.manager = audit.New() + } + if a.userManager == nil { + a.userManager = user.New() + } +} + +func (a AuditLogsDataMasking) Run(ctx job.Context, params job.Parameters) error { + logger := ctx.GetLogger() + logger.Info("GDPR audit logs data masking job started") + a.init() + username, err := a.parseParams(params) + if err != nil { + return err + } + logger.Infof("Masking log entries for a user: %s", username) + return a.manager.UpdateUsername(ctx.SystemContext(), username, a.userManager.GenerateCheckSum(username)) +} + +func (a AuditLogsDataMasking) parseParams(params job.Parameters) (string, error) { + value, exist := params[UserNameParam] + if !exist { + return "", fmt.Errorf("param %s not found", UserNameParam) + } + str, ok := value.(string) + if !ok { + return "", fmt.Errorf("the value of %s isn't string", UserNameParam) + } + return str, nil +} diff --git a/src/jobservice/job/impl/gdpr/audit_logs_data_masking_test.go b/src/jobservice/job/impl/gdpr/audit_logs_data_masking_test.go new file mode 100644 index 00000000000..d928a88cffc --- /dev/null +++ b/src/jobservice/job/impl/gdpr/audit_logs_data_masking_test.go @@ -0,0 +1,67 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gdpr + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/goharbor/harbor/src/jobservice/job" + mockjobservice "github.com/goharbor/harbor/src/testing/jobservice" + "github.com/goharbor/harbor/src/testing/pkg/audit" + "github.com/goharbor/harbor/src/testing/pkg/user" +) + +func TestAuditLogsCleanupJobShouldRetry(t *testing.T) { + rep := &AuditLogsDataMasking{} + assert.True(t, rep.ShouldRetry()) +} + +func TestAuditLogsCleanupJobValidateParams(t *testing.T) { + const validUsername = "user" + var ( + manager = &audit.Manager{} + userManager = &user.Manager{} + ) + + rep := &AuditLogsDataMasking{ + manager: manager, + userManager: userManager, + } + err := rep.Validate(nil) + // parameters are required + assert.Error(t, err) + err = rep.Validate(job.Parameters{}) + // no required username parameter + assert.Error(t, err) + validParams := job.Parameters{ + "username": "user", + } + err = rep.Validate(validParams) + // parameters are valid + assert.Nil(t, err) + + ctx := &mockjobservice.MockJobContext{} + logger := &mockjobservice.MockJobLogger{} + + ctx.On("GetLogger").Return(logger) + userManager.On("GenerateCheckSum", validUsername).Return("hash") + manager.On("UpdateUsername", context.TODO(), validUsername, "hash").Return(nil) + + err = rep.Run(ctx, validParams) + assert.Nil(t, err) +} diff --git a/src/jobservice/job/known_jobs.go b/src/jobservice/job/known_jobs.go index 5ff43497c6b..5944decfa08 100644 --- a/src/jobservice/job/known_jobs.go +++ b/src/jobservice/job/known_jobs.go @@ -44,6 +44,8 @@ const ( ExecSweepVendorType = "EXECUTION_SWEEP" // ScanAllVendorType: the name of the scan all job ScanAllVendorType = "SCAN_ALL" + // AuditLogsGDPRCompliantVendorType : the name of the job which makes audit logs table GDPR-compliant + AuditLogsGDPRCompliantVendorType = "AUDIT_LOGS_GDPR_COMPLIANT" ) var ( diff --git a/src/jobservice/runtime/bootstrap.go b/src/jobservice/runtime/bootstrap.go index aad3837bb70..bd2aabd7c7e 100644 --- a/src/jobservice/runtime/bootstrap.go +++ b/src/jobservice/runtime/bootstrap.go @@ -24,6 +24,8 @@ import ( "syscall" "time" + "github.com/goharbor/harbor/src/jobservice/job/impl/gdpr" + "github.com/gomodule/redigo/redis" "github.com/goharbor/harbor/src/jobservice/api" @@ -327,11 +329,12 @@ func (bs *Bootstrap) loadAndRunRedisWorkerPool( // In v2.2 we migrate the scheduled replication, garbage collection and scan all to // the scheduler mechanism, the following three jobs are kept for the legacy jobs // and they can be removed after several releases - "IMAGE_REPLICATE": (*legacy.ReplicationScheduler)(nil), - "IMAGE_GC": (*legacy.GarbageCollectionScheduler)(nil), - "IMAGE_SCAN_ALL": (*legacy.ScanAllScheduler)(nil), - job.SystemArtifactCleanupVendorType: (*systemartifact.Cleanup)(nil), - job.ExecSweepVendorType: (*task.SweepJob)(nil), + "IMAGE_REPLICATE": (*legacy.ReplicationScheduler)(nil), + "IMAGE_GC": (*legacy.GarbageCollectionScheduler)(nil), + "IMAGE_SCAN_ALL": (*legacy.ScanAllScheduler)(nil), + job.SystemArtifactCleanupVendorType: (*systemartifact.Cleanup)(nil), + job.ExecSweepVendorType: (*task.SweepJob)(nil), + job.AuditLogsGDPRCompliantVendorType: (*gdpr.AuditLogsDataMasking)(nil), }); err != nil { // exit return nil, err diff --git a/src/lib/config/metadata/metadatalist.go b/src/lib/config/metadata/metadatalist.go index 535226bc837..f1ceb938bc2 100644 --- a/src/lib/config/metadata/metadatalist.go +++ b/src/lib/config/metadata/metadatalist.go @@ -181,6 +181,7 @@ var ( {Name: common.CacheExpireHours, Scope: SystemScope, Group: BasicGroup, EnvKey: "CACHE_EXPIRE_HOURS", DefaultValue: "24", ItemType: &IntType{}, Editable: false, Description: `The expire hours for cache`}, {Name: common.GDPRDeleteUser, Scope: SystemScope, Group: GDPRGroup, EnvKey: "GDPR_DELETE_USER", DefaultValue: "false", ItemType: &BoolType{}, Editable: false, Description: `The flag indicates if a user should be deleted compliant with GDPR.`}, + {Name: common.GDPRAuditLogs, Scope: SystemScope, Group: GDPRGroup, EnvKey: "GDPR_AUDIT_LOGS", DefaultValue: "false", ItemType: &BoolType{}, Editable: false, Description: `The flag indicates if an audit logs of a deleted user should be GDPR compliant.`}, {Name: common.AuditLogForwardEndpoint, Scope: UserScope, Group: BasicGroup, EnvKey: "AUDIT_LOG_FORWARD_ENDPOINT", DefaultValue: "", ItemType: &StringType{}, Editable: false, Description: `The endpoint to forward the audit log.`}, {Name: common.SkipAuditLogDatabase, Scope: UserScope, Group: BasicGroup, EnvKey: "SKIP_LOG_AUDIT_DATABASE", DefaultValue: "false", ItemType: &BoolType{}, Editable: false, Description: `The option to skip audit log in database`}, diff --git a/src/lib/config/models/model.go b/src/lib/config/models/model.go index 6b21fda8084..51a9c0c2cbc 100644 --- a/src/lib/config/models/model.go +++ b/src/lib/config/models/model.go @@ -98,4 +98,5 @@ type GroupConf struct { type GDPRSetting struct { DeleteUser bool `json:"user_delete,omitempty"` + AuditLogs bool `json:"audit_logs"` } diff --git a/src/lib/config/userconfig.go b/src/lib/config/userconfig.go index e22b438f7eb..0192e6f5854 100644 --- a/src/lib/config/userconfig.go +++ b/src/lib/config/userconfig.go @@ -186,6 +186,7 @@ func GDPRSetting(ctx context.Context) (*cfgModels.GDPRSetting, error) { } return &cfgModels.GDPRSetting{ DeleteUser: DefaultMgr().Get(ctx, common.GDPRDeleteUser).GetBool(), + AuditLogs: DefaultMgr().Get(ctx, common.GDPRAuditLogs).GetBool(), }, nil } diff --git a/src/pkg/audit/dao/dao.go b/src/pkg/audit/dao/dao.go index 019ffe6d5bd..3a816b6c458 100644 --- a/src/pkg/audit/dao/dao.go +++ b/src/pkg/audit/dao/dao.go @@ -42,6 +42,8 @@ type DAO interface { Delete(ctx context.Context, id int64) (err error) // Purge the audit log Purge(ctx context.Context, retentionHour int, includeOperations []string, dryRun bool) (int64, error) + // UpdateUsername replaces username in matched records + UpdateUsername(ctx context.Context, username string, usernameReplace string) error } // New returns an instance of the default DAO @@ -57,6 +59,15 @@ var allowedMaps = map[string]interface{}{ type dao struct{} +func (d *dao) UpdateUsername(ctx context.Context, username string, usernameReplace string) error { + o, err := orm.FromContext(ctx) + if err != nil { + return err + } + _, err = o.Raw("UPDATE audit_log SET username = ? WHERE username = ?", usernameReplace, username).Exec() + return err +} + // Purge delete expired audit log func (*dao) Purge(ctx context.Context, retentionHour int, includeOperations []string, dryRun bool) (int64, error) { ormer, err := orm.FromContext(ctx) diff --git a/src/pkg/audit/manager.go b/src/pkg/audit/manager.go index b81a89bfe09..136d23ff77a 100644 --- a/src/pkg/audit/manager.go +++ b/src/pkg/audit/manager.go @@ -40,6 +40,8 @@ type Manager interface { Delete(ctx context.Context, id int64) (err error) // Purge delete the audit log with retention hours Purge(ctx context.Context, retentionHour int, includeOperations []string, dryRun bool) (int64, error) + // UpdateUsername Replace all log records username with its hash + UpdateUsername(ctx context.Context, username string, replaceWith string) error } // New returns a default implementation of Manager @@ -53,6 +55,10 @@ type manager struct { dao dao.DAO } +func (m *manager) UpdateUsername(ctx context.Context, username string, replaceWith string) error { + return m.dao.UpdateUsername(ctx, username, replaceWith) +} + // Count ... func (m *manager) Count(ctx context.Context, query *q.Query) (int64, error) { return m.dao.Count(ctx, query) diff --git a/src/pkg/user/manager.go b/src/pkg/user/manager.go index f6420f4fad4..5efb05578ff 100644 --- a/src/pkg/user/manager.go +++ b/src/pkg/user/manager.go @@ -63,6 +63,8 @@ type Manager interface { // put the id in the pointer of user model, if it does exist, return the user's profile. // This is used for ldap and uaa authentication, such the user can have an ID in Harbor. Onboard(ctx context.Context, user *commonmodels.User) error + // GenerateCheckSum generates truncated crc32 checksum from a given string + GenerateCheckSum(in string) string } // New returns a default implementation of Manager @@ -111,9 +113,9 @@ func (m *manager) DeleteGDPR(ctx context.Context, id int) error { if err != nil { return err } - u.Username = fmt.Sprintf("%s#%d", checkSum(u.Username), u.UserID) - u.Email = fmt.Sprintf("%s#%d", checkSum(u.Email), u.UserID) - u.Realname = fmt.Sprintf("%s#%d", checkSum(u.Realname), u.UserID) + u.Username = fmt.Sprintf("%s#%d", m.GenerateCheckSum(u.Username), u.UserID) + u.Email = fmt.Sprintf("%s#%d", m.GenerateCheckSum(u.Email), u.UserID) + u.Realname = fmt.Sprintf("%s#%d", m.GenerateCheckSum(u.Realname), u.UserID) u.Deleted = true return m.dao.Update(ctx, u, "username", "email", "realname", "deleted") } @@ -231,13 +233,14 @@ func excludeDefaultAdmin(query *q.Query) (qu *q.Query) { return query } +// GenerateCheckSum generates checksum for a given string +func (m *manager) GenerateCheckSum(str string) string { + return fmt.Sprintf("%08x", crc32.Checksum([]byte(str), crc32.IEEETable)) +} + func injectPasswd(u *commonmodels.User, password string) { salt := utils.GenerateRandomString() u.Password = utils.Encrypt(password, salt, utils.SHA256) u.Salt = salt u.PasswordVersion = utils.SHA256 } - -func checkSum(str string) string { - return fmt.Sprintf("%08x", crc32.Checksum([]byte(str), crc32.IEEETable)) -} diff --git a/src/pkg/user/manager_test.go b/src/pkg/user/manager_test.go index 32714be5faa..166513149b7 100644 --- a/src/pkg/user/manager_test.go +++ b/src/pkg/user/manager_test.go @@ -65,9 +65,9 @@ func (m *mgrTestSuite) TestUserDeleteGDPR() { m.dao.On("Update", mock.Anything, testifymock.MatchedBy( func(u *models.User) bool { return u.UserID == 123 && - u.Email == fmt.Sprintf("%s#%d", checkSum("existing@mytest.com"), existingUser.UserID) && - u.Username == fmt.Sprintf("%s#%d", checkSum("existing"), existingUser.UserID) && - u.Realname == fmt.Sprintf("%s#%d", checkSum("RealName"), existingUser.UserID) && + u.Email == fmt.Sprintf("%s#%d", m.mgr.GenerateCheckSum("existing@mytest.com"), existingUser.UserID) && + u.Username == fmt.Sprintf("%s#%d", m.mgr.GenerateCheckSum("existing"), existingUser.UserID) && + u.Realname == fmt.Sprintf("%s#%d", m.mgr.GenerateCheckSum("RealName"), existingUser.UserID) && u.Deleted == true }), "username", diff --git a/src/testing/pkg/audit/dao/dao.go b/src/testing/pkg/audit/dao/dao.go index bbfce5e3134..97665fdaaa3 100644 --- a/src/testing/pkg/audit/dao/dao.go +++ b/src/testing/pkg/audit/dao/dao.go @@ -155,6 +155,20 @@ func (_m *DAO) Purge(ctx context.Context, retentionHour int, includeOperations [ return r0, r1 } +// UpdateUsername provides a mock function with given fields: ctx, username, usernameReplace +func (_m *DAO) UpdateUsername(ctx context.Context, username string, usernameReplace string) error { + ret := _m.Called(ctx, username, usernameReplace) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) error); ok { + r0 = rf(ctx, username, usernameReplace) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // NewDAO creates a new instance of DAO. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewDAO(t interface { diff --git a/src/testing/pkg/audit/manager.go b/src/testing/pkg/audit/manager.go index 38fbd38a7d9..1efcd06dfa6 100644 --- a/src/testing/pkg/audit/manager.go +++ b/src/testing/pkg/audit/manager.go @@ -154,6 +154,20 @@ func (_m *Manager) Purge(ctx context.Context, retentionHour int, includeOperatio return r0, r1 } +// UpdateUsername provides a mock function with given fields: ctx, username, replaceWith +func (_m *Manager) UpdateUsername(ctx context.Context, username string, replaceWith string) error { + ret := _m.Called(ctx, username, replaceWith) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) error); ok { + r0 = rf(ctx, username, replaceWith) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // NewManager creates a new instance of Manager. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewManager(t interface { diff --git a/src/testing/pkg/user/manager.go b/src/testing/pkg/user/manager.go index aec51e17a96..83b4f9d7726 100644 --- a/src/testing/pkg/user/manager.go +++ b/src/testing/pkg/user/manager.go @@ -102,6 +102,20 @@ func (_m *Manager) DeleteGDPR(ctx context.Context, id int) error { return r0 } +// GenerateCheckSum provides a mock function with given fields: in +func (_m *Manager) GenerateCheckSum(in string) string { + ret := _m.Called(in) + + var r0 string + if rf, ok := ret.Get(0).(func(string) string); ok { + r0 = rf(in) + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + // Get provides a mock function with given fields: ctx, id func (_m *Manager) Get(ctx context.Context, id int) (*commonmodels.User, error) { ret := _m.Called(ctx, id) diff --git a/tests/apitests/python/test_job_service_dashboard.py b/tests/apitests/python/test_job_service_dashboard.py index 6fdc8460e23..fd71c4c5a1a 100644 --- a/tests/apitests/python/test_job_service_dashboard.py +++ b/tests/apitests/python/test_job_service_dashboard.py @@ -33,7 +33,7 @@ def setUp(self): self.registry = Registry() self.scan_all = ScanAll() self.schedule = Schedule() - self.job_types = [ "GARBAGE_COLLECTION", "PURGE_AUDIT_LOG", "P2P_PREHEAT", "IMAGE_SCAN", "REPLICATION", "RETENTION", "SCAN_DATA_EXPORT", "SCHEDULER", "SLACK", "SYSTEM_ARTIFACT_CLEANUP", "WEBHOOK", "EXECUTION_SWEEP"] + self.job_types = [ "GARBAGE_COLLECTION", "PURGE_AUDIT_LOG", "P2P_PREHEAT", "IMAGE_SCAN", "REPLICATION", "RETENTION", "SCAN_DATA_EXPORT", "SCHEDULER", "SLACK", "SYSTEM_ARTIFACT_CLEANUP", "WEBHOOK", "EXECUTION_SWEEP", "AUDIT_LOGS_GDPR_COMPLIANT"] self.cron_type = "Custom" self.cron = "0 0 0 * * 0" From 553c85eed04aea65db3336be45a628ce022148fb Mon Sep 17 00:00:00 2001 From: Chlins Zhang Date: Tue, 21 Nov 2023 15:36:06 +0800 Subject: [PATCH 10/11] fix: increase beego max memory and upload size (#19578) 1. Increase the default beego max memory and upload size from 32GB to 128GB. 2. Support customize the two beego configs from env. Signed-off-by: chlins Co-authored-by: Wang Yan --- src/common/const.go | 10 ++++++++++ src/core/main.go | 8 ++++++-- src/lib/config/metadata/metadatalist.go | 9 ++++++++- src/lib/config/systemconfig.go | 10 ++++++++++ 4 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/common/const.go b/src/common/const.go index a00601fcc40..64b38da33a8 100644 --- a/src/common/const.go +++ b/src/common/const.go @@ -231,4 +231,14 @@ const ( QuotaUpdateProvider = "quota_update_provider" // IllegalCharsInUsername is the illegal chars in username IllegalCharsInUsername = `,"~#%$` + + // Beego web config + // BeegoMaxMemoryBytes is the max memory(bytes) of the beego web config + BeegoMaxMemoryBytes = "beego_max_memory_bytes" + // DefaultBeegoMaxMemoryBytes sets default max memory to 128GB + DefaultBeegoMaxMemoryBytes = 1 << 37 + // BeegoMaxUploadSizeBytes is the max upload size(bytes) of the beego web config + BeegoMaxUploadSizeBytes = "beego_max_upload_size_bytes" + // DefaultBeegoMaxUploadSizeBytes sets default max upload size to 128GB + DefaultBeegoMaxUploadSizeBytes = 1 << 37 ) diff --git a/src/core/main.go b/src/core/main.go index 78fe8e171ee..b660ea012be 100644 --- a/src/core/main.go +++ b/src/core/main.go @@ -127,8 +127,6 @@ func main() { web.BConfig.WebConfig.Session.SessionOn = true web.BConfig.WebConfig.Session.SessionName = config.SessionCookieName - web.BConfig.MaxMemory = 1 << 35 // (32GB) - web.BConfig.MaxUploadSize = 1 << 35 // (32GB) // the core db used for beego session redisCoreURL := os.Getenv("_REDIS_URL_CORE") if len(redisCoreURL) > 0 { @@ -163,6 +161,12 @@ func main() { log.Info("initializing configurations...") config.Init() log.Info("configurations initialization completed") + + // default beego max memory and max upload size is 128GB, consider from some AI related image would be large, + // also support customize it from the environment variables if the default value cannot satisfy some scenarios. + web.BConfig.MaxMemory = config.GetBeegoMaxMemoryBytes() + web.BConfig.MaxUploadSize = config.GetBeegoMaxUploadSizeBytes() + metricCfg := config.Metric() if metricCfg.Enabled { metric.RegisterCollectors() diff --git a/src/lib/config/metadata/metadatalist.go b/src/lib/config/metadata/metadatalist.go index f1ceb938bc2..dd2a7f67cde 100644 --- a/src/lib/config/metadata/metadatalist.go +++ b/src/lib/config/metadata/metadatalist.go @@ -14,7 +14,11 @@ package metadata -import "github.com/goharbor/harbor/src/common" +import ( + "fmt" + + "github.com/goharbor/harbor/src/common" +) // Item - Configure item include default value, type, env name type Item struct { @@ -193,5 +197,8 @@ var ( {Name: common.BannerMessage, Scope: UserScope, Group: BasicGroup, EnvKey: "BANNER_MESSAGE", DefaultValue: "", ItemType: &StringType{}, Editable: true, Description: `The customized banner message for the UI`}, {Name: common.QuotaUpdateProvider, Scope: SystemScope, Group: BasicGroup, EnvKey: "QUOTA_UPDATE_PROVIDER", DefaultValue: "db", ItemType: &StringType{}, Editable: false, Description: `The provider for updating quota, 'db' or 'redis' is supported`}, + + {Name: common.BeegoMaxMemoryBytes, Scope: SystemScope, Group: BasicGroup, EnvKey: "BEEGO_MAX_MEMORY_BYTES", DefaultValue: fmt.Sprintf("%d", common.DefaultBeegoMaxMemoryBytes), ItemType: &Int64Type{}, Editable: false, Description: `The bytes for limiting the beego max memory, default is 128GB`}, + {Name: common.BeegoMaxUploadSizeBytes, Scope: SystemScope, Group: BasicGroup, EnvKey: "BEEGO_MAX_UPLOAD_SIZE_BYTES", DefaultValue: fmt.Sprintf("%d", common.DefaultBeegoMaxUploadSizeBytes), ItemType: &Int64Type{}, Editable: false, Description: `The bytes for limiting the beego max upload size, default it 128GB`}, } ) diff --git a/src/lib/config/systemconfig.go b/src/lib/config/systemconfig.go index 1af4d8474ca..7019d9466b1 100644 --- a/src/lib/config/systemconfig.go +++ b/src/lib/config/systemconfig.go @@ -132,6 +132,16 @@ func GetQuotaUpdateProvider() string { return DefaultMgr().Get(backgroundCtx, common.QuotaUpdateProvider).GetString() } +// GetBeegoMaxMemoryBytes returns the max memory bytes of beego config +func GetBeegoMaxMemoryBytes() int64 { + return DefaultMgr().Get(backgroundCtx, common.BeegoMaxMemoryBytes).GetInt64() +} + +// GetBeegoMaxUploadSizeBytes returns the max upload size bytes of beego config +func GetBeegoMaxUploadSizeBytes() int64 { + return DefaultMgr().Get(backgroundCtx, common.BeegoMaxUploadSizeBytes).GetInt64() +} + // WithTrivy returns a bool value to indicate if Harbor's deployed with Trivy. func WithTrivy() bool { return DefaultMgr().Get(backgroundCtx, common.WithTrivy).GetBool() From 17c17ffd8ed1cd7a278f6ef3448f08dc93b7657e Mon Sep 17 00:00:00 2001 From: MinerYang Date: Tue, 21 Nov 2023 19:23:16 +0800 Subject: [PATCH 11/11] bump golang to 1.21.4 (#19601) Signed-off-by: yminer --- .github/workflows/CI.yml | 10 +++++----- .github/workflows/build-package.yml | 2 +- .github/workflows/conformance_test.yml | 2 +- CONTRIBUTING.md | 2 ++ Makefile | 2 +- make/photon/registry/Dockerfile.binary | 2 +- make/photon/trivy-adapter/Dockerfile.binary | 2 +- make/photon/trivy-adapter/builder.sh | 2 +- tests/ci/distro_installer.sh | 4 ++-- 9 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 529ea4fc753..0d367ec8764 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -44,7 +44,7 @@ jobs: - name: Set up Go 1.21 uses: actions/setup-go@v1 with: - go-version: 1.21.3 + go-version: 1.21.4 id: go - uses: actions/checkout@v3 with: @@ -105,7 +105,7 @@ jobs: - name: Set up Go 1.21 uses: actions/setup-go@v1 with: - go-version: 1.21.3 + go-version: 1.21.4 id: go - uses: actions/checkout@v3 with: @@ -160,7 +160,7 @@ jobs: - name: Set up Go 1.21 uses: actions/setup-go@v1 with: - go-version: 1.21.3 + go-version: 1.21.4 id: go - uses: actions/checkout@v3 with: @@ -215,7 +215,7 @@ jobs: - name: Set up Go 1.21 uses: actions/setup-go@v1 with: - go-version: 1.21.3 + go-version: 1.21.4 id: go - uses: actions/checkout@v3 with: @@ -268,7 +268,7 @@ jobs: - name: Set up Go 1.21 uses: actions/setup-go@v1 with: - go-version: 1.21.3 + go-version: 1.21.4 id: go - uses: actions/checkout@v3 with: diff --git a/.github/workflows/build-package.yml b/.github/workflows/build-package.yml index 25d6bc783fd..259c2cf380c 100644 --- a/.github/workflows/build-package.yml +++ b/.github/workflows/build-package.yml @@ -26,7 +26,7 @@ jobs: - name: Set up Go 1.21 uses: actions/setup-go@v1 with: - go-version: 1.21.3 + go-version: 1.21.4 id: go - name: Setup Docker uses: docker-practice/actions-setup-docker@master diff --git a/.github/workflows/conformance_test.yml b/.github/workflows/conformance_test.yml index 7570e2aa750..0245681edb0 100644 --- a/.github/workflows/conformance_test.yml +++ b/.github/workflows/conformance_test.yml @@ -28,7 +28,7 @@ jobs: - name: Set up Go 1.21 uses: actions/setup-go@v1 with: - go-version: 1.21.3 + go-version: 1.21.4 id: go - uses: actions/checkout@v3 with: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 64856c01fba..b15588af212 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -164,6 +164,8 @@ Harbor backend is written in [Go](http://golang.org/). If you don't have a Harbo | 2.7 | 1.19.4 | | 2.8 | 1.20.6 | | 2.9 | 1.21.3 | +| 2.10 | 1.21.4 | + Ensure your GOPATH and PATH have been configured in accordance with the Go environment instructions. diff --git a/Makefile b/Makefile index f105cd590b0..a0725465a69 100644 --- a/Makefile +++ b/Makefile @@ -140,7 +140,7 @@ GOINSTALL=$(GOCMD) install GOTEST=$(GOCMD) test GODEP=$(GOTEST) -i GOFMT=gofmt -w -GOBUILDIMAGE=golang:1.21.3 +GOBUILDIMAGE=golang:1.21.4 GOBUILDPATHINCONTAINER=/harbor # go build diff --git a/make/photon/registry/Dockerfile.binary b/make/photon/registry/Dockerfile.binary index 58478e5049f..d77f133fe40 100644 --- a/make/photon/registry/Dockerfile.binary +++ b/make/photon/registry/Dockerfile.binary @@ -1,4 +1,4 @@ -FROM golang:1.21.3 +FROM golang:1.21.4 ENV DISTRIBUTION_DIR /go/src/github.com/docker/distribution ENV BUILDTAGS include_oss include_gcs diff --git a/make/photon/trivy-adapter/Dockerfile.binary b/make/photon/trivy-adapter/Dockerfile.binary index 051a296e432..9c6d3d4ab02 100644 --- a/make/photon/trivy-adapter/Dockerfile.binary +++ b/make/photon/trivy-adapter/Dockerfile.binary @@ -1,4 +1,4 @@ -FROM golang:1.21.3 +FROM golang:1.21.4 ADD . /go/src/github.com/aquasecurity/harbor-scanner-trivy/ WORKDIR /go/src/github.com/aquasecurity/harbor-scanner-trivy/ diff --git a/make/photon/trivy-adapter/builder.sh b/make/photon/trivy-adapter/builder.sh index 7f98e75e10f..7b51c2673cf 100755 --- a/make/photon/trivy-adapter/builder.sh +++ b/make/photon/trivy-adapter/builder.sh @@ -19,7 +19,7 @@ TEMP=$(mktemp -d ${TMPDIR-/tmp}/trivy-adapter.XXXXXX) git clone https://github.com/aquasecurity/harbor-scanner-trivy.git $TEMP cd $TEMP; git checkout $VERSION; cd - -echo "Building Trivy adapter binary based on golang:1.21.3..." +echo "Building Trivy adapter binary based on golang:1.21.4..." cp Dockerfile.binary $TEMP docker build -f $TEMP/Dockerfile.binary -t trivy-adapter-golang $TEMP diff --git a/tests/ci/distro_installer.sh b/tests/ci/distro_installer.sh index ae5ac7ab3fb..f97449d846e 100755 --- a/tests/ci/distro_installer.sh +++ b/tests/ci/distro_installer.sh @@ -3,5 +3,5 @@ set -x set -e -sudo make package_online GOBUILDTAGS="include_oss include_gcs" VERSIONTAG=dev-gitaction PKGVERSIONTAG=dev-gitaction UIVERSIONTAG=dev-gitaction GOBUILDIMAGE=golang:1.21.3 COMPILETAG=compile_golangimage TRIVYFLAG=true HTTPPROXY= PULL_BASE_FROM_DOCKERHUB=false -sudo make package_offline GOBUILDTAGS="include_oss include_gcs" VERSIONTAG=dev-gitaction PKGVERSIONTAG=dev-gitaction UIVERSIONTAG=dev-gitaction GOBUILDIMAGE=golang:1.21.3 COMPILETAG=compile_golangimage TRIVYFLAG=true HTTPPROXY= PULL_BASE_FROM_DOCKERHUB=false +sudo make package_online GOBUILDTAGS="include_oss include_gcs" VERSIONTAG=dev-gitaction PKGVERSIONTAG=dev-gitaction UIVERSIONTAG=dev-gitaction GOBUILDIMAGE=golang:1.21.4 COMPILETAG=compile_golangimage TRIVYFLAG=true HTTPPROXY= PULL_BASE_FROM_DOCKERHUB=false +sudo make package_offline GOBUILDTAGS="include_oss include_gcs" VERSIONTAG=dev-gitaction PKGVERSIONTAG=dev-gitaction UIVERSIONTAG=dev-gitaction GOBUILDIMAGE=golang:1.21.4 COMPILETAG=compile_golangimage TRIVYFLAG=true HTTPPROXY= PULL_BASE_FROM_DOCKERHUB=false