diff --git a/CHANGELOG.md b/CHANGELOG.md index 78b25b7f78f..bf337acfca1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 + +## \[2.7.5\] - 2023-10-09 + +### Added + +- Temporary workaround to fix corrupted zip file + () + ## \[2.7.4\] - 2023-10-06 diff --git a/cvat-cli/requirements/base.txt b/cvat-cli/requirements/base.txt index 828b45da493..383315d20fc 100644 --- a/cvat-cli/requirements/base.txt +++ b/cvat-cli/requirements/base.txt @@ -1,3 +1,3 @@ -cvat-sdk~=2.7.4 +cvat-sdk~=2.7.5 Pillow>=10.0.1 setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/cvat-cli/src/cvat_cli/version.py b/cvat-cli/src/cvat_cli/version.py index df1ba35501b..0fcd1c9993a 100644 --- a/cvat-cli/src/cvat_cli/version.py +++ b/cvat-cli/src/cvat_cli/version.py @@ -1 +1 @@ -VERSION = "2.7.4" +VERSION = "2.7.5" diff --git a/cvat-core/src/download.worker.js b/cvat-core/src/download.worker.js index b3e48789357..411a49abd9a 100644 --- a/cvat-core/src/download.worker.js +++ b/cvat-core/src/download.worker.js @@ -13,6 +13,7 @@ onmessage = (e) => { .then((response) => { postMessage({ responseData: response.data, + headers: response.headers, id: e.data.id, isSuccess: true, }); diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index 978b2c4bf61..7296764e092 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -199,7 +199,7 @@ class WorkerWrappedAxios { if (e.data.id in requests) { try { if (e.data.isSuccess) { - requests[e.data.id].resolve(e.data.responseData); + requests[e.data.id].resolve({ data: e.data.responseData, headers: e.data.headers }); } else { requests[e.data.id].reject(new AxiosError(e.data.message, e.data.code)); } @@ -1495,7 +1495,7 @@ async function getImageContext(jid: number, frame: number): Promise } } -async function getData(jid: number, chunk: number, quality: ChunkQuality): Promise { +async function getData(jid: number, chunk: number, quality: ChunkQuality, retry = 0): Promise { const { backendAPI } = config; try { @@ -1509,7 +1509,29 @@ async function getData(jid: number, chunk: number, quality: ChunkQuality): Promi responseType: 'arraybuffer', }); - return response; + const contentLength = +(response.headers || {})['content-length']; + if (Number.isInteger(contentLength) && response.data.byteLength < +contentLength) { + if (retry < 10) { + // corrupted zip tmp workaround + // if content length more than received byteLength, request the chunk again + // and log this error + setTimeout(() => { + throw new Error( + `Truncated chunk, try: ${retry}. Job: ${jid}, chunk: ${chunk}, quality: ${quality}. ` + + `Body size: ${response.data.byteLength}`, + ); + }); + return await getData(jid, chunk, quality, retry + 1); + } + + // not to try anymore, throw explicit error + throw new Error( + `Truncated chunk. Job: ${jid}, chunk: ${chunk}, quality: ${quality}. ` + + `Body size: ${response.data.byteLength}`, + ); + } + + return response.data; } catch (errorData) { throw generateError(errorData); } diff --git a/cvat-sdk/gen/generate.sh b/cvat-sdk/gen/generate.sh index 36720a226b5..d83e3c6b43c 100755 --- a/cvat-sdk/gen/generate.sh +++ b/cvat-sdk/gen/generate.sh @@ -8,7 +8,7 @@ set -e GENERATOR_VERSION="v6.0.1" -VERSION="2.7.4" +VERSION="2.7.5" LIB_NAME="cvat_sdk" LAYER1_LIB_NAME="${LIB_NAME}/api_client" DST_DIR="$(cd "$(dirname -- "$0")/.." && pwd)" diff --git a/cvat-ui/src/components/create-cloud-storage-page/manifests-manager.tsx b/cvat-ui/src/components/create-cloud-storage-page/manifests-manager.tsx index 2a5809b5268..2d2419a6c48 100644 --- a/cvat-ui/src/components/create-cloud-storage-page/manifests-manager.tsx +++ b/cvat-ui/src/components/create-cloud-storage-page/manifests-manager.tsx @@ -100,6 +100,7 @@ export default function ManifestsManager(props: Props): JSX.Element { > onChangeManifestPath(event.target.value, idx)} /> diff --git a/cvat/__init__.py b/cvat/__init__.py index d6e2e2e3adf..191ad96a2a2 100644 --- a/cvat/__init__.py +++ b/cvat/__init__.py @@ -4,6 +4,6 @@ from cvat.utils.version import get_version -VERSION = (2, 7, 4, 'final', 0) +VERSION = (2, 7, 5, 'final', 0) __version__ = get_version(VERSION) diff --git a/cvat/requirements/base.in b/cvat/requirements/base.in index ef0dfedd5a4..810cfe5fd64 100644 --- a/cvat/requirements/base.in +++ b/cvat/requirements/base.in @@ -27,7 +27,6 @@ django-rq==2.3.2 django-sendfile2==0.7.0 Django~=4.2.1 djangorestframework~=3.14.0 -dnspython==2.2.0 drf-spectacular>=0.26.2 furl==2.1.0 google-cloud-storage==1.42.0 diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index 2718bb916b1..1eb97880917 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:f4b1cfa1e64a1d036a99deca36db609d665a7bd8 +# SHA1:efa1135fd6eb8ff1c784d632ee43797415192c71 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -116,8 +116,6 @@ djangorestframework==3.14.0 # -r cvat/requirements/base.in # dj-rest-auth # drf-spectacular -dnspython==2.2.0 - # via -r cvat/requirements/base.in drf-spectacular==0.26.2 # via -r cvat/requirements/base.in easyprocess==1.1 diff --git a/cvat/schema.yml b/cvat/schema.yml index 14a6213833c..89947c52441 100644 --- a/cvat/schema.yml +++ b/cvat/schema.yml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: CVAT REST API - version: 2.7.4 + version: 2.7.5 description: REST API for Computer Vision Annotation Tool (CVAT) termsOfService: https://www.google.com/policies/terms/ contact: diff --git a/docker-compose.yml b/docker-compose.yml index ebe36ef7ab8..b857c95ddf2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,7 +34,7 @@ services: cvat_server: container_name: cvat_server - image: cvat/server:${CVAT_VERSION:-v2.7.4} + image: cvat/server:${CVAT_VERSION:-v2.7.5} restart: always depends_on: - cvat_redis @@ -73,7 +73,7 @@ services: cvat_utils: container_name: cvat_utils - image: cvat/server:${CVAT_VERSION:-v2.7.4} + image: cvat/server:${CVAT_VERSION:-v2.7.5} restart: always depends_on: - cvat_redis @@ -98,7 +98,7 @@ services: cvat_worker_import: container_name: cvat_worker_import - image: cvat/server:${CVAT_VERSION:-v2.7.4} + image: cvat/server:${CVAT_VERSION:-v2.7.5} restart: always depends_on: - cvat_redis @@ -121,7 +121,7 @@ services: cvat_worker_export: container_name: cvat_worker_export - image: cvat/server:${CVAT_VERSION:-v2.7.4} + image: cvat/server:${CVAT_VERSION:-v2.7.5} restart: always depends_on: - cvat_redis @@ -144,7 +144,7 @@ services: cvat_worker_annotation: container_name: cvat_worker_annotation - image: cvat/server:${CVAT_VERSION:-v2.7.4} + image: cvat/server:${CVAT_VERSION:-v2.7.5} restart: always depends_on: - cvat_redis @@ -167,7 +167,7 @@ services: cvat_worker_webhooks: container_name: cvat_worker_webhooks - image: cvat/server:${CVAT_VERSION:-v2.7.4} + image: cvat/server:${CVAT_VERSION:-v2.7.5} restart: always depends_on: - cvat_redis @@ -191,7 +191,7 @@ services: cvat_worker_quality_reports: container_name: cvat_worker_quality_reports - image: cvat/server:${CVAT_VERSION:-v2.7.4} + image: cvat/server:${CVAT_VERSION:-v2.7.5} restart: always depends_on: - cvat_redis @@ -213,7 +213,7 @@ services: cvat_worker_analytics_reports: container_name: cvat_worker_analytics_reports - image: cvat/server:${CVAT_VERSION:-v2.7.4} + image: cvat/server:${CVAT_VERSION:-v2.7.5} restart: always depends_on: - cvat_redis @@ -236,7 +236,7 @@ services: cvat_ui: container_name: cvat_ui - image: cvat/ui:${CVAT_VERSION:-v2.7.4} + image: cvat/ui:${CVAT_VERSION:-v2.7.5} restart: always depends_on: - cvat_server diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index 338ab21baf5..c3b38efc712 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -104,7 +104,7 @@ cvat: additionalVolumeMounts: [] replicas: 1 image: cvat/server - tag: v2.7.4 + tag: v2.7.5 imagePullPolicy: Always permissionFix: enabled: true @@ -128,7 +128,7 @@ cvat: frontend: replicas: 1 image: cvat/ui - tag: v2.7.4 + tag: v2.7.5 imagePullPolicy: Always labels: {} # test: test diff --git a/site/content/en/docs/manual/basics/attach-cloud-storage.md b/site/content/en/docs/manual/basics/attach-cloud-storage.md index 68cd24b2255..3c40e26f746 100644 --- a/site/content/en/docs/manual/basics/attach-cloud-storage.md +++ b/site/content/en/docs/manual/basics/attach-cloud-storage.md @@ -5,8 +5,10 @@ weight: 23 description: 'Instructions on how to attach cloud storage using UI' --- -In CVAT you can use [AWS-S3](using-aws-s3), [Azure Blob Container](#using-azure-blob-container) -and [Google cloud](#using-google-cloud-storage) storages to store image datasets for your tasks. +In CVAT you can use **AWS S3**, **Azure Blob Container** +and **Google Cloud** storages to import and export +image datasets for your tasks. + See: @@ -29,7 +31,7 @@ See: - [Create a bucket](#create-a-bucket-2) - [Create a container](#create-a-container) - [Upload data](#upload-data-2) - - [SAS token](#sas-token) + - [SAS token and connection string](#sas-token-and-connection-string) - [Personal use](#personal-use) - [Attach Azure Blob Container](#attach-azure-blob-container) - [Prepare the dataset](#prepare-the-dataset) @@ -115,13 +117,13 @@ Fill in the following fields: | CVAT | AWS S3 | | ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Display name** | Preferred display name for your storage. | +| **Display name** | Preferred display name for your storage. | | **Description** | (Optional) Add description of storage. | | **Provider** | From drop-down list select **AWS S3**. | | **Bucket name** | Name of the [Bucket](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingBucket). | | **Authorization type** | Depends on the bucket setup:
  • **Key id and secret access key pair**: available on [IAM](https://console.aws.amazon.com/iamv2/home?#/users).
  • **Anonymous access**: for anonymous access. Public access to the bucket must be enabled. | | **Region** | (Optional) Choose a region from the list or add a new one. For more information, see [**Available locations**](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). | -| **Manifests** | Click **+ Add manifest** and enter the name of the manifest file with an extension. For example: `manifest.jsonl`. | +| **Manifests** | Click **+ Add manifest** and enter the name of the manifest file with an extension. For example: `manifest.jsonl`. | @@ -404,14 +406,12 @@ Fill in the following fields: | **Provider** | From drop-down list select **Azure Blob Container**. | | **Container name`** | Name of the cloud storage container. | | **Authorization type** | Depends on the container setup.
    **[Account name and SAS token](https://docs.microsoft.com/en-us/azure/cognitive-services/translator/document-translation/create-sas-tokens?tabs=blobs)**:
    • **Account name** enter storage account name.
    • **SAS token** is located in the **Shared access signature** section of your [Storage account](#sas-token).
    . **[Anonymous access](https://docs.microsoft.com/en-us/azure/storage/blobs/anonymous-read-access-configure?tabs=portal)**: for anonymous access **Allow enabling public access on containers** must be enabled. | -| **Manifests** | Click **+ Add manifest** and enter the name of the manifest file with an extention. For example: `manifest.jsonl`. | +| **Manifests** | Click **+ Add manifest** and enter the name of the manifest file with an extention. For example: `manifest.jsonl`. | After filling in all the fields, click **Submit**. - - ## Prepare the dataset For example, the dataset is [The Oxford-IIIT Pet Dataset](https://www.robots.ox.ac.uk/~vgg/data/pets/): diff --git a/tests/cypress/e2e/actions_tasks3/case_105_cloud_storage.js b/tests/cypress/e2e/actions_tasks3/case_105_cloud_storage.js index 8bacf46fd94..6657a6e851d 100644 --- a/tests/cypress/e2e/actions_tasks3/case_105_cloud_storage.js +++ b/tests/cypress/e2e/actions_tasks3/case_105_cloud_storage.js @@ -40,13 +40,26 @@ context('Cloud storage.', () => { cy.get($el).should('exist'); }); }); + }); + + it('Check "Cloud Storage" manifest field.', () => { // Check add/remove manifest file cy.get('.cvat-add-manifest-button').should('be.visible').click(); - cy.get('[placeholder="manifest.jsonl"]').should('exist').should('have.attr', 'value', ''); - cy.get('[placeholder="manifest.jsonl"]').type(dummyData.manifest); - cy.get('[placeholder="manifest.jsonl"]').should('have.attr', 'value', dummyData.manifest); + cy.get('.cvat-cloud-storage-manifest-field').should('exist').should('have.attr', 'value', ''); + cy.get('.cvat-cloud-storage-manifest-field').type(dummyData.manifest); + cy.get('.cvat-cloud-storage-manifest-field').should('have.attr', 'value', dummyData.manifest); + cy.get('[data-icon="delete"]').should('be.visible').click(); + cy.get('.cvat-cloud-storage-manifest-field').should('not.exist'); + + // Check we can't add non-jsonl file + cy.get('.cvat-add-manifest-button').should('be.visible').click(); + cy.get('.cvat-cloud-storage-manifest-field').type('manifest.json'); + cy.get('.cvat-cloud-storage-manifest-field').should('have.attr', 'value', 'manifest.json'); + cy.get('.cvat-cloud-storage-form').within(() => { + cy.contains('Manifest file must have .jsonl extension').should('exist'); + }); cy.get('[data-icon="delete"]').should('be.visible').click(); - cy.get('[placeholder="manifest.jsonl"]').should('not.exist'); + cy.get('.cvat-cloud-storage-manifest-field').should('not.exist'); }); it('Check "AWS S3" provider fields.', () => {