Release v2.7.5

cvat-ai · Oct 9, 2023 · 7956a18 · 7956a18
2 parents 2ec5c44 + 20db599
commit 7956a18
Show file tree

Hide file tree

Showing 15 changed files with 77 additions and 35 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -16,6 +16,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 <!-- scriv-insert-here -->
 
+<a id='changelog-2.7.5'></a>
+## \[2.7.5\] - 2023-10-09
+
+### Added
+
+- Temporary workaround to fix corrupted zip file
+  (<https://github.com/opencv/cvat/pull/6965>)
+
 <a id='changelog-2.7.4'></a>
 ## \[2.7.4\] - 2023-10-06
 

diff --git a/cvat-cli/requirements/base.txt b/cvat-cli/requirements/base.txt
@@ -1,3 +1,3 @@
-cvat-sdk~=2.7.4
+cvat-sdk~=2.7.5
 Pillow>=10.0.1
 setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability
diff --git a/cvat-cli/src/cvat_cli/version.py b/cvat-cli/src/cvat_cli/version.py
@@ -1 +1 @@
-VERSION = "2.7.4"
+VERSION = "2.7.5"
diff --git a/cvat-core/src/download.worker.js b/cvat-core/src/download.worker.js
@@ -13,6 +13,7 @@ onmessage = (e) => {
         .then((response) => {
             postMessage({
                 responseData: response.data,
+                headers: response.headers,
                 id: e.data.id,
                 isSuccess: true,
             });

diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts
@@ -199,7 +199,7 @@ class WorkerWrappedAxios {
             if (e.data.id in requests) {
                 try {
                     if (e.data.isSuccess) {
-                        requests[e.data.id].resolve(e.data.responseData);
+                        requests[e.data.id].resolve({ data: e.data.responseData, headers: e.data.headers });
                     } else {
                         requests[e.data.id].reject(new AxiosError(e.data.message, e.data.code));
                     }
@@ -1495,7 +1495,7 @@ async function getImageContext(jid: number, frame: number): Promise<ArrayBuffer>
     }
 }
 
-async function getData(jid: number, chunk: number, quality: ChunkQuality): Promise<ArrayBuffer> {
+async function getData(jid: number, chunk: number, quality: ChunkQuality, retry = 0): Promise<ArrayBuffer> {
     const { backendAPI } = config;
 
     try {
@@ -1509,7 +1509,29 @@ async function getData(jid: number, chunk: number, quality: ChunkQuality): Promi
             responseType: 'arraybuffer',
         });
 
-        return response;
+        const contentLength = +(response.headers || {})['content-length'];
+        if (Number.isInteger(contentLength) && response.data.byteLength < +contentLength) {
+            if (retry < 10) {
+                // corrupted zip tmp workaround
+                // if content length more than received byteLength, request the chunk again
+                // and log this error
+                setTimeout(() => {
+                    throw new Error(
+                        `Truncated chunk, try: ${retry}. Job: ${jid}, chunk: ${chunk}, quality: ${quality}. ` +
+                        `Body size: ${response.data.byteLength}`,
+                    );
+                });
+                return await getData(jid, chunk, quality, retry + 1);
+            }
+
+            // not to try anymore, throw explicit error
+            throw new Error(
+                `Truncated chunk. Job: ${jid}, chunk: ${chunk}, quality: ${quality}. ` +
+                `Body size: ${response.data.byteLength}`,
+            );
+        }
+
+        return response.data;
     } catch (errorData) {
         throw generateError(errorData);
     }

diff --git a/cvat-sdk/gen/generate.sh b/cvat-sdk/gen/generate.sh
@@ -8,7 +8,7 @@ set -e
 
 GENERATOR_VERSION="v6.0.1"
 
-VERSION="2.7.4"
+VERSION="2.7.5"
 LIB_NAME="cvat_sdk"
 LAYER1_LIB_NAME="${LIB_NAME}/api_client"
 DST_DIR="$(cd "$(dirname -- "$0")/.." && pwd)"

diff --git a/cvat-ui/src/components/create-cloud-storage-page/manifests-manager.tsx b/cvat-ui/src/components/create-cloud-storage-page/manifests-manager.tsx
@@ -100,6 +100,7 @@ export default function ManifestsManager(props: Props): JSX.Element {
                                             >
                                                 <Input
                                                     placeholder='manifest.jsonl'
+                                                    className='cvat-cloud-storage-manifest-field'
                                                     onChange={(event) => onChangeManifestPath(event.target.value, idx)}
                                                 />
                                             </Form.Item>

diff --git a/cvat/__init__.py b/cvat/__init__.py
@@ -4,6 +4,6 @@
 
 from cvat.utils.version import get_version
 
-VERSION = (2, 7, 4, 'final', 0)
+VERSION = (2, 7, 5, 'final', 0)
 
 __version__ = get_version(VERSION)
diff --git a/cvat/requirements/base.in b/cvat/requirements/base.in
@@ -27,7 +27,6 @@ django-rq==2.3.2
 django-sendfile2==0.7.0
 Django~=4.2.1
 djangorestframework~=3.14.0
-dnspython==2.2.0
 drf-spectacular>=0.26.2
 furl==2.1.0
 google-cloud-storage==1.42.0

diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt
@@ -1,4 +1,4 @@
-# SHA1:f4b1cfa1e64a1d036a99deca36db609d665a7bd8
+# SHA1:efa1135fd6eb8ff1c784d632ee43797415192c71
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -116,8 +116,6 @@ djangorestframework==3.14.0
     #   -r cvat/requirements/base.in
     #   dj-rest-auth
     #   drf-spectacular
-dnspython==2.2.0
-    # via -r cvat/requirements/base.in
 drf-spectacular==0.26.2
     # via -r cvat/requirements/base.in
 easyprocess==1.1

diff --git a/cvat/schema.yml b/cvat/schema.yml
@@ -1,7 +1,7 @@
 openapi: 3.0.3
 info:
   title: CVAT REST API
-  version: 2.7.4
+  version: 2.7.5
   description: REST API for Computer Vision Annotation Tool (CVAT)
   termsOfService: https://www.google.com/policies/terms/
   contact:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -34,7 +34,7 @@ services:
 
   cvat_server:
     container_name: cvat_server
-    image: cvat/server:${CVAT_VERSION:-v2.7.4}
+    image: cvat/server:${CVAT_VERSION:-v2.7.5}
     restart: always
     depends_on:
       - cvat_redis
@@ -73,7 +73,7 @@ services:
 
   cvat_utils:
     container_name: cvat_utils
-    image: cvat/server:${CVAT_VERSION:-v2.7.4}
+    image: cvat/server:${CVAT_VERSION:-v2.7.5}
     restart: always
     depends_on:
       - cvat_redis
@@ -98,7 +98,7 @@ services:
 
   cvat_worker_import:
     container_name: cvat_worker_import
-    image: cvat/server:${CVAT_VERSION:-v2.7.4}
+    image: cvat/server:${CVAT_VERSION:-v2.7.5}
     restart: always
     depends_on:
       - cvat_redis
@@ -121,7 +121,7 @@ services:
 
   cvat_worker_export:
     container_name: cvat_worker_export
-    image: cvat/server:${CVAT_VERSION:-v2.7.4}
+    image: cvat/server:${CVAT_VERSION:-v2.7.5}
     restart: always
     depends_on:
       - cvat_redis
@@ -144,7 +144,7 @@ services:
 
   cvat_worker_annotation:
     container_name: cvat_worker_annotation
-    image: cvat/server:${CVAT_VERSION:-v2.7.4}
+    image: cvat/server:${CVAT_VERSION:-v2.7.5}
     restart: always
     depends_on:
       - cvat_redis
@@ -167,7 +167,7 @@ services:
 
   cvat_worker_webhooks:
     container_name: cvat_worker_webhooks
-    image: cvat/server:${CVAT_VERSION:-v2.7.4}
+    image: cvat/server:${CVAT_VERSION:-v2.7.5}
     restart: always
     depends_on:
       - cvat_redis
@@ -191,7 +191,7 @@ services:
 
   cvat_worker_quality_reports:
     container_name: cvat_worker_quality_reports
-    image: cvat/server:${CVAT_VERSION:-v2.7.4}
+    image: cvat/server:${CVAT_VERSION:-v2.7.5}
     restart: always
     depends_on:
       - cvat_redis
@@ -213,7 +213,7 @@ services:
 
   cvat_worker_analytics_reports:
     container_name: cvat_worker_analytics_reports
-    image: cvat/server:${CVAT_VERSION:-v2.7.4}
+    image: cvat/server:${CVAT_VERSION:-v2.7.5}
     restart: always
     depends_on:
       - cvat_redis
@@ -236,7 +236,7 @@ services:
 
   cvat_ui:
     container_name: cvat_ui
-    image: cvat/ui:${CVAT_VERSION:-v2.7.4}
+    image: cvat/ui:${CVAT_VERSION:-v2.7.5}
     restart: always
     depends_on:
       - cvat_server

diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml
@@ -104,7 +104,7 @@ cvat:
       additionalVolumeMounts: []
     replicas: 1
     image: cvat/server
-    tag: v2.7.4
+    tag: v2.7.5
     imagePullPolicy: Always
     permissionFix:
       enabled: true
@@ -128,7 +128,7 @@ cvat:
   frontend:
     replicas: 1
     image: cvat/ui
-    tag: v2.7.4
+    tag: v2.7.5
     imagePullPolicy: Always
     labels: {}
     #  test: test

diff --git a/site/content/en/docs/manual/basics/attach-cloud-storage.md b/site/content/en/docs/manual/basics/attach-cloud-storage.md
@@ -5,8 +5,10 @@ weight: 23
 description: 'Instructions on how to attach cloud storage using UI'
 ---
 
-In CVAT you can use [AWS-S3](using-aws-s3), [Azure Blob Container](#using-azure-blob-container)
-and [Google cloud](#using-google-cloud-storage) storages to store image datasets for your tasks.
+In CVAT you can use **AWS S3**, **Azure Blob Container**
+and **Google Cloud** storages to import and export
+image datasets for your tasks.
+
 
 See:
 
@@ -29,7 +31,7 @@ See:
   - [Create a bucket](#create-a-bucket-2)
   - [Create a container](#create-a-container)
   - [Upload data](#upload-data-2)
-  - [SAS token](#sas-token)
+  - [SAS token and connection string](#sas-token-and-connection-string)
   - [Personal use](#personal-use)
   - [Attach Azure Blob Container](#attach-azure-blob-container)
 - [Prepare the dataset](#prepare-the-dataset)
@@ -115,13 +117,13 @@ Fill in the following fields:
 
 | CVAT                   | AWS S3                                                                                                                                                                                                                                              |
 | ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **Display name**       | Preferred display name for your storage.                                                                                                                                                                                                                              |
+| **Display name**       | Preferred display name for your storage.                                                                                                                                                                                                            |
 | **Description**        | (Optional) Add description of storage.                                                                                                                                                                                                              |
 | **Provider**           | From drop-down list select **AWS S3**.                                                                                                                                                                                                              |
 | **Bucket name**        | Name of the [Bucket](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingBucket).                                                                                                                                                            |
 | **Authorization type** | Depends on the bucket setup: <br><li>**Key id and secret access key pair**: available on [IAM](https://console.aws.amazon.com/iamv2/home?#/users). <br><li>**Anonymous access**: for anonymous access. Public access to the bucket must be enabled. |
 | **Region**             | (Optional) Choose a region from the list or add a new one. For more information, see [**Available locations**](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions).               |
-| **Manifests**          | Click **+ Add manifest** and enter the name of the manifest file with an extension. For example: `manifest.jsonl`.                                                                                                                                   |
+| **Manifests**          | Click **+ Add manifest** and enter the name of the manifest file with an extension. For example: `manifest.jsonl`.                                                                                                                                  |
 
 <!--lint enable maximum-line-length-->
 
@@ -404,14 +406,12 @@ Fill in the following fields:
 | **Provider**           | From drop-down list select **Azure Blob Container**.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
 | **Container name`**    | Name of the cloud storage container.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
 | **Authorization type** | Depends on the container setup. <br>**[Account name and SAS token](https://docs.microsoft.com/en-us/azure/cognitive-services/translator/document-translation/create-sas-tokens?tabs=blobs)**: <ul><li>**Account name** enter storage account name. <li>**SAS token** is located in the **Shared access signature** section of your [Storage account](#sas-token).</ul>. **[Anonymous access](https://docs.microsoft.com/en-us/azure/storage/blobs/anonymous-read-access-configure?tabs=portal)**: for anonymous access **Allow enabling public access on containers** must be enabled. |
-| **Manifests**          | Click **+ Add manifest** and enter the name of the manifest file with an extention. For example: `manifest.jsonl`.                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
+| **Manifests**          | Click **+ Add manifest** and enter the name of the manifest file with an extention. For example: `manifest.jsonl`.                                                                                                                                                                                                                                                                                                                                                                                                                                                                     |
 
 <!--lint enable maximum-line-length-->
 
 After filling in all the fields, click **Submit**.
 
-
-
 ## Prepare the dataset
 
 For example, the dataset is [The Oxford-IIIT Pet Dataset](https://www.robots.ox.ac.uk/~vgg/data/pets/):

diff --git a/tests/cypress/e2e/actions_tasks3/case_105_cloud_storage.js b/tests/cypress/e2e/actions_tasks3/case_105_cloud_storage.js
@@ -40,13 +40,26 @@ context('Cloud storage.', () => {
                     cy.get($el).should('exist');
                 });
             });
+        });
+
+        it('Check "Cloud Storage" manifest field.', () => {
             // Check add/remove manifest file
             cy.get('.cvat-add-manifest-button').should('be.visible').click();
-            cy.get('[placeholder="manifest.jsonl"]').should('exist').should('have.attr', 'value', '');
-            cy.get('[placeholder="manifest.jsonl"]').type(dummyData.manifest);
-            cy.get('[placeholder="manifest.jsonl"]').should('have.attr', 'value', dummyData.manifest);
+            cy.get('.cvat-cloud-storage-manifest-field').should('exist').should('have.attr', 'value', '');
+            cy.get('.cvat-cloud-storage-manifest-field').type(dummyData.manifest);
+            cy.get('.cvat-cloud-storage-manifest-field').should('have.attr', 'value', dummyData.manifest);
+            cy.get('[data-icon="delete"]').should('be.visible').click();
+            cy.get('.cvat-cloud-storage-manifest-field').should('not.exist');
+
+            // Check we can't add non-jsonl file
+            cy.get('.cvat-add-manifest-button').should('be.visible').click();
+            cy.get('.cvat-cloud-storage-manifest-field').type('manifest.json');
+            cy.get('.cvat-cloud-storage-manifest-field').should('have.attr', 'value', 'manifest.json');
+            cy.get('.cvat-cloud-storage-form').within(() => {
+                cy.contains('Manifest file must have .jsonl extension').should('exist');
+            });
             cy.get('[data-icon="delete"]').should('be.visible').click();
-            cy.get('[placeholder="manifest.jsonl"]').should('not.exist');
+            cy.get('.cvat-cloud-storage-manifest-field').should('not.exist');
         });
 
         it('Check "AWS S3" provider fields.', () => {