diff --git a/cycode/cli/consts.py b/cycode/cli/consts.py index 37ef2298..58924c2d 100644 --- a/cycode/cli/consts.py +++ b/cycode/cli/consts.py @@ -222,13 +222,15 @@ FILE_MAX_SIZE_LIMIT_IN_BYTES = 5000000 PRESIGNED_LINK_UPLOADED_ZIP_MAX_SIZE_LIMIT_IN_BYTES = 5 * 1024 * 1024 * 1024 # 5 GB (S3 presigned POST limit) -PRESIGNED_UPLOAD_SCAN_TYPES = {SAST_SCAN_TYPE, SECRET_SCAN_TYPE} +PRESIGNED_UPLOAD_SCAN_TYPES = {SAST_SCAN_TYPE} +# Secret scans use the previous (batched / API-upload) flow by default. The presigned S3 async flow is +# opt-in via the SECRET_SCAN_ASYNC_ENV_VAR_NAME env var; see should_use_presigned_upload. +SECRET_SCAN_ASYNC_ENV_VAR_NAME = 'CYCODE_SECRET_SCAN_ASYNC' DEFAULT_ZIP_MAX_SIZE_LIMIT_IN_BYTES = 20 * 1024 * 1024 ZIP_MAX_SIZE_LIMIT_IN_BYTES = { SCA_SCAN_TYPE: 200 * 1024 * 1024, SAST_SCAN_TYPE: PRESIGNED_LINK_UPLOADED_ZIP_MAX_SIZE_LIMIT_IN_BYTES, - SECRET_SCAN_TYPE: PRESIGNED_LINK_UPLOADED_ZIP_MAX_SIZE_LIMIT_IN_BYTES, } # scan in batches diff --git a/cycode/cli/files_collector/zip_documents.py b/cycode/cli/files_collector/zip_documents.py index 7927bdc6..4f66429f 100644 --- a/cycode/cli/files_collector/zip_documents.py +++ b/cycode/cli/files_collector/zip_documents.py @@ -6,13 +6,17 @@ from cycode.cli.exceptions import custom_exceptions from cycode.cli.files_collector.models.in_memory_zip import InMemoryZip from cycode.cli.models import Document +from cycode.cli.utils.scan_utils import should_use_presigned_upload from cycode.logger import get_logger logger = get_logger('ZIP') def _validate_zip_file_size(scan_type: str, zip_file_size: int) -> None: - max_size_limit = consts.ZIP_MAX_SIZE_LIMIT_IN_BYTES.get(scan_type, consts.DEFAULT_ZIP_MAX_SIZE_LIMIT_IN_BYTES) + if should_use_presigned_upload(scan_type): + max_size_limit = consts.PRESIGNED_LINK_UPLOADED_ZIP_MAX_SIZE_LIMIT_IN_BYTES + else: + max_size_limit = consts.ZIP_MAX_SIZE_LIMIT_IN_BYTES.get(scan_type, consts.DEFAULT_ZIP_MAX_SIZE_LIMIT_IN_BYTES) if zip_file_size > max_size_limit: raise custom_exceptions.ZipTooLargeError(max_size_limit) diff --git a/cycode/cli/utils/scan_utils.py b/cycode/cli/utils/scan_utils.py index 819a4116..21131343 100644 --- a/cycode/cli/utils/scan_utils.py +++ b/cycode/cli/utils/scan_utils.py @@ -7,6 +7,7 @@ from cycode.cli import consts from cycode.cli.cli_types import SeverityOption +from cycode.config import parse_bool if TYPE_CHECKING: from cycode.cli.models import LocalScanResult @@ -33,7 +34,11 @@ def is_cycodeignore_allowed_by_scan_config(ctx: typer.Context) -> bool: def should_use_presigned_upload(scan_type: str) -> bool: - return scan_type in consts.PRESIGNED_UPLOAD_SCAN_TYPES + if scan_type in consts.PRESIGNED_UPLOAD_SCAN_TYPES: + return True + if scan_type == consts.SECRET_SCAN_TYPE: + return parse_bool(os.getenv(consts.SECRET_SCAN_ASYNC_ENV_VAR_NAME)) + return False def generate_unique_scan_id() -> UUID: diff --git a/cycode/config.py b/cycode/config.py index f4306b31..b17dadaa 100644 --- a/cycode/config.py +++ b/cycode/config.py @@ -19,11 +19,18 @@ def get_val_as_string(key: str) -> str: return configuration.get(key) +_TRUTHY_STRING_VALUES = {'true', '1', 'yes', 'y', 'on', 'enabled'} + + +def parse_bool(value: Optional[str]) -> bool: + return value is not None and value.lower() in _TRUTHY_STRING_VALUES + + def get_val_as_bool(key: str, default: bool = False) -> bool: if key not in configuration: return default - return configuration[key].lower() in {'true', '1', 'yes', 'y', 'on', 'enabled'} + return parse_bool(configuration[key]) def get_val_as_int(key: str) -> Optional[int]: diff --git a/tests/cli/commands/scan/test_code_scanner.py b/tests/cli/commands/scan/test_code_scanner.py index 8b4a30b3..11d0fd46 100644 --- a/tests/cli/commands/scan/test_code_scanner.py +++ b/tests/cli/commands/scan/test_code_scanner.py @@ -168,14 +168,16 @@ def test_entrypoint_cycode_not_added_for_single_file( @pytest.mark.parametrize( - ('scan_type', 'command_scan_type', 'sync_option', 'expect_presigned'), + ('scan_type', 'command_scan_type', 'sync_option', 'secret_async_env', 'expect_presigned'), [ # SAST keeps uploading directly to S3 via a presigned URL (regression guard for the new sync gate). - (consts.SAST_SCAN_TYPE, 'path', False, True), - # Async secret scans now upload as a single file directly to S3 via a presigned URL. - (consts.SECRET_SCAN_TYPE, 'path', False, True), - # A --sync secret scan must stay on the batched inline path and never build one giant zip. - (consts.SECRET_SCAN_TYPE, 'path', True, False), + (consts.SAST_SCAN_TYPE, 'path', False, False, True), + # Secret scans use the previous batched flow by default (presigned async is opt-in). + (consts.SECRET_SCAN_TYPE, 'path', False, False, False), + # With CYCODE_SECRET_SCAN_ASYNC enabled, secret scans upload as a single file directly to S3. + (consts.SECRET_SCAN_TYPE, 'path', False, True, True), + # A --sync secret scan must stay on the batched inline path even when async is enabled. + (consts.SECRET_SCAN_TYPE, 'path', True, True, False), ], ) @patch('cycode.cli.apps.scan.code_scanner.print_local_scan_results') @@ -192,8 +194,13 @@ def test_scan_documents_routes_upload_by_scan_type_and_sync( scan_type: str, command_scan_type: str, sync_option: bool, + secret_async_env: bool, expect_presigned: bool, + monkeypatch: pytest.MonkeyPatch, ) -> None: + if secret_async_env: + monkeypatch.setenv(consts.SECRET_SCAN_ASYNC_ENV_VAR_NAME, 'true') + mock_presigned_upload.return_value = ([], []) mock_batched_scan.return_value = ([], []) diff --git a/tests/cli/commands/scan/test_commit_range_scanner.py b/tests/cli/commands/scan/test_commit_range_scanner.py index a4a6c58b..8b8cc54b 100644 --- a/tests/cli/commands/scan/test_commit_range_scanner.py +++ b/tests/cli/commands/scan/test_commit_range_scanner.py @@ -1,5 +1,7 @@ from unittest.mock import MagicMock, Mock, patch +import pytest + from cycode.cli import consts from cycode.cli.apps.scan.commit_range_scanner import _scan_commit_range_documents from cycode.cli.exceptions import custom_exceptions @@ -25,7 +27,10 @@ def test_commit_range_scan_falls_back_to_api_when_presigned_upload_raises_wrappe mock_print: Mock, mock_handle_exception: Mock, mock_report_status: Mock, + monkeypatch: pytest.MonkeyPatch, ) -> None: + # Secret uses the presigned flow only when async is opted in. + monkeypatch.setenv(consts.SECRET_SCAN_ASYNC_ENV_VAR_NAME, 'true') # SlowUploadConnectionError is a CycodeError, not a requests.RequestException — the presigned # commit-range fallback must still catch it and retry via the Cycode API. mock_v4_async.side_effect = custom_exceptions.SlowUploadConnectionError diff --git a/tests/cli/utils/__init__.py b/tests/cli/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/utils/test_scan_utils.py b/tests/cli/utils/test_scan_utils.py new file mode 100644 index 00000000..d2bf0438 --- /dev/null +++ b/tests/cli/utils/test_scan_utils.py @@ -0,0 +1,46 @@ +import pytest + +from cycode.cli import consts +from cycode.cli.exceptions import custom_exceptions +from cycode.cli.files_collector.zip_documents import _validate_zip_file_size +from cycode.cli.utils.scan_utils import should_use_presigned_upload + + +def test_sast_always_uses_presigned_upload() -> None: + assert should_use_presigned_upload(consts.SAST_SCAN_TYPE) is True + + +def test_secret_does_not_use_presigned_upload_by_default(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv(consts.SECRET_SCAN_ASYNC_ENV_VAR_NAME, raising=False) + assert should_use_presigned_upload(consts.SECRET_SCAN_TYPE) is False + + +@pytest.mark.parametrize('env_value', ['true', 'True', '1', 'yes', 'y', 'on', 'enabled']) +def test_secret_uses_presigned_upload_when_env_enabled(monkeypatch: pytest.MonkeyPatch, env_value: str) -> None: + monkeypatch.setenv(consts.SECRET_SCAN_ASYNC_ENV_VAR_NAME, env_value) + assert should_use_presigned_upload(consts.SECRET_SCAN_TYPE) is True + + +@pytest.mark.parametrize('env_value', ['false', '0', 'no', '', 'off']) +def test_secret_ignores_non_truthy_env_values(monkeypatch: pytest.MonkeyPatch, env_value: str) -> None: + monkeypatch.setenv(consts.SECRET_SCAN_ASYNC_ENV_VAR_NAME, env_value) + assert should_use_presigned_upload(consts.SECRET_SCAN_TYPE) is False + + +def test_sca_never_uses_presigned_upload() -> None: + assert should_use_presigned_upload(consts.SCA_SCAN_TYPE) is False + + +def test_secret_zip_size_limit_uses_default_by_default(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv(consts.SECRET_SCAN_ASYNC_ENV_VAR_NAME, raising=False) + # A zip just above the default 20 MB limit must be rejected on the previous (batched) flow. + with pytest.raises(custom_exceptions.ZipTooLargeError): + _validate_zip_file_size(consts.SECRET_SCAN_TYPE, consts.DEFAULT_ZIP_MAX_SIZE_LIMIT_IN_BYTES + 1) + + +def test_secret_zip_size_limit_uses_presigned_limit_when_env_enabled(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(consts.SECRET_SCAN_ASYNC_ENV_VAR_NAME, 'true') + # The same zip fits under the 5 GB presigned limit when async is enabled. + _validate_zip_file_size(consts.SECRET_SCAN_TYPE, consts.DEFAULT_ZIP_MAX_SIZE_LIMIT_IN_BYTES + 1) + with pytest.raises(custom_exceptions.ZipTooLargeError): + _validate_zip_file_size(consts.SECRET_SCAN_TYPE, consts.PRESIGNED_LINK_UPLOADED_ZIP_MAX_SIZE_LIMIT_IN_BYTES + 1)