Проект разделен на два репозитория

This commit is contained in:
mi
2026-08-14 15:42:45 +03:00
parent e06a77ee1d
commit bbef7a30c9
521 changed files with 2597 additions and 2302 deletions
@@ -0,0 +1,58 @@
from __future__ import annotations
import asyncio
import os
from pathlib import Path
from sqlalchemy import pool
from sqlalchemy.ext.asyncio import async_engine_from_config
from alembic import context
from app.repository import postgres_ssl_context
def migration_url() -> str:
path = os.getenv("BITRIX_SYNC_MIGRATION_DATABASE_URL_FILE")
if not path:
raise RuntimeError("BITRIX_SYNC_MIGRATION_DATABASE_URL_FILE is required")
value = Path(path).read_text(encoding="utf-8").rstrip("\r\n")
if not value:
raise RuntimeError("Bitrix migration database URL file is empty")
return value
def run_migrations_offline() -> None:
context.configure(
url=migration_url(),
target_metadata=None,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
async def run_async_migrations() -> None:
configuration = context.config.get_section(context.config.config_ini_section) or {}
configuration["sqlalchemy.url"] = migration_url()
engine = async_engine_from_config(
configuration,
prefix="sqlalchemy.",
poolclass=pool.NullPool,
connect_args={"ssl": postgres_ssl_context()},
)
def run_sync_migrations(connection) -> None:
context.configure(connection=connection, target_metadata=None, compare_type=True)
with context.begin_transaction():
context.run_migrations()
async with engine.connect() as connection:
await connection.run_sync(run_sync_migrations)
await engine.dispose()
if context.is_offline_mode():
run_migrations_offline()
else:
asyncio.run(run_async_migrations())
@@ -0,0 +1,20 @@
"""Preserve the legacy connectivity-stub Alembic revision.
Revision ID: 0001_sync_baseline
Revises:
"""
from collections.abc import Sequence
revision: str = "0001_sync_baseline"
down_revision: str | None = None
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""The legacy connectivity stub owned no runtime tables."""
def downgrade() -> None:
"""The legacy connectivity stub owned no runtime tables."""
@@ -0,0 +1,356 @@
"""Create durable bitrix_sync schema and contracts.
Revision ID: 0001_bitrix_sync_full
Revises: 0001_sync_baseline
Create Date: 2026-08-06
"""
from collections.abc import Sequence
from alembic import op
revision: str = "0001_bitrix_sync_full"
down_revision: str | None = "0001_sync_baseline"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _execute_script(script: str) -> None:
"""Execute simple DDL statements separately for asyncpg compatibility."""
for statement in script.split(";"):
if statement.strip():
op.execute(statement)
def upgrade() -> None:
op.execute("CREATE EXTENSION IF NOT EXISTS pgcrypto")
op.execute("CREATE SCHEMA IF NOT EXISTS bitrix_sync")
_execute_script(
"""
CREATE TABLE bitrix_sync.workflow_instances (
id uuid PRIMARY KEY,
workflow_type varchar(64) NOT NULL CHECK (workflow_type IN
('contact.map_or_create','contact.update','contact.deactivate','contact.rebind',
'contact.webhook','contact.reconciliation','alert.reconciliation')),
user_id uuid,
external_id varchar(128),
state varchar(24) NOT NULL CHECK (state IN
('created','running','waiting_crm','waiting_retry','waiting_manual',
'succeeded','failed','cancelled')),
current_step varchar(64) NOT NULL,
source_task_id uuid UNIQUE,
deadline_at timestamptz NOT NULL,
outcome varchar(64),
completed_at timestamptz,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX ix_workflow_claim
ON bitrix_sync.workflow_instances(state,updated_at)
WHERE state IN ('created','running','waiting_crm','waiting_retry');
CREATE TABLE bitrix_sync.entity_external_mapping (
id uuid PRIMARY KEY,
entity_type varchar(64) NOT NULL,
entity_id uuid NOT NULL,
external_system varchar(32) NOT NULL DEFAULT 'bitrix24'
CHECK (external_system='bitrix24'),
external_entity_type varchar(32) NOT NULL DEFAULT 'contact'
CHECK (external_entity_type='contact'),
external_id varchar(128) NOT NULL,
status varchar(16) NOT NULL CHECK (status IN ('active','closed','broken')),
opened_at timestamptz NOT NULL,
closed_at timestamptz,
close_reason varchar(64),
workflow_id uuid REFERENCES bitrix_sync.workflow_instances(id),
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
CHECK ((status='active' AND closed_at IS NULL) OR
(status IN ('closed','broken')
AND (closed_at IS NOT NULL OR close_reason IS NOT NULL)))
);
CREATE UNIQUE INDEX uq_mapping_active_entity
ON bitrix_sync.entity_external_mapping(external_system,entity_type,entity_id)
WHERE status='active';
CREATE UNIQUE INDEX uq_mapping_active_external
ON bitrix_sync.entity_external_mapping
(external_system,external_entity_type,external_id)
WHERE status='active';
CREATE INDEX ix_mapping_history
ON bitrix_sync.entity_external_mapping(entity_id,opened_at DESC);
CREATE TABLE bitrix_sync.crm_commands (
id uuid PRIMARY KEY,
workflow_id uuid NOT NULL REFERENCES bitrix_sync.workflow_instances(id),
command_type varchar(40) NOT NULL CHECK (command_type IN
('duplicate_find','contact_get','contact_add','contact_update',
'citizenship_fields_get','contact_incremental_list',
'alert_get','alert_add','alert_update',
'rebind_target_get','rebind_old_get')),
safe_request jsonb NOT NULL DEFAULT '{}',
status varchar(24) NOT NULL CHECK (status IN
('pending','leased','in_flight','succeeded','retry','retry_wait',
'uncertain','reconcile','dead_letter','permanent','rate_limited')),
attempt_count integer NOT NULL DEFAULT 0 CHECK (attempt_count>=0),
next_attempt_at timestamptz NOT NULL DEFAULT now(),
locked_by varchar(128),
locked_until timestamptz,
lease_token uuid,
batch_id uuid,
correlation_id uuid,
safe_response jsonb,
safe_error_code varchar(64),
http_status integer,
completed_at timestamptz,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX ix_crm_command_claim
ON bitrix_sync.crm_commands(status,next_attempt_at,created_at);
CREATE INDEX ix_crm_command_workflow
ON bitrix_sync.crm_commands(workflow_id,created_at);
CREATE TABLE bitrix_sync.webhook_inbox (
id uuid PRIMARY KEY,
receiver_type varchar(16) NOT NULL CHECK (receiver_type IN ('contact','alert')),
event_type varchar(64) NOT NULL,
event_id varchar(255),
source_timestamp timestamptz,
external_entity_id varchar(128) NOT NULL,
dedup_fingerprint varchar(64),
source_ip inet,
status varchar(16) NOT NULL CHECK (status IN
('received','coalesced','processing','processed','retry_wait','dead_letter')),
coalesced_count integer NOT NULL DEFAULT 1,
attempt_count integer NOT NULL DEFAULT 0,
next_attempt_at timestamptz NOT NULL DEFAULT now(),
locked_by varchar(128),
locked_until timestamptz,
lease_token uuid,
received_at timestamptz NOT NULL,
last_received_at timestamptz NOT NULL,
processed_at timestamptz,
safe_error_code varchar(64)
);
CREATE UNIQUE INDEX uq_webhook_event_id
ON bitrix_sync.webhook_inbox(receiver_type,event_id) WHERE event_id IS NOT NULL;
CREATE INDEX ix_webhook_claim
ON bitrix_sync.webhook_inbox(status,next_attempt_at,received_at);
CREATE UNIQUE INDEX uq_webhook_active_entity
ON bitrix_sync.webhook_inbox(receiver_type,external_entity_id)
WHERE status IN ('received','processing','retry_wait');
"""
)
_execute_script(
"""
CREATE SEQUENCE bitrix_sync.business_alert_number_seq;
CREATE TABLE bitrix_sync.business_alerts (
id uuid PRIMARY KEY,
alert_number bigint NOT NULL DEFAULT nextval('bitrix_sync.business_alert_number_seq'),
fingerprint varchar(64) NOT NULL,
alert_type varchar(64) NOT NULL,
severity varchar(16) NOT NULL CHECK (severity IN ('info','warning','critical')),
app_user_id uuid,
current_external_id varchar(128),
selected_external_id varchar(128),
candidate_external_ids text[] NOT NULL DEFAULT '{}',
remote_item_id varchar(128),
remote_stage_id varchar(128),
previous_alert_id uuid REFERENCES bitrix_sync.business_alerts(id),
workflow_id uuid REFERENCES bitrix_sync.workflow_instances(id),
status varchar(24) NOT NULL CHECK (status IN
('open','in_progress','resolved','closed_without_resolution','remote_missing')),
occurrence_count integer NOT NULL DEFAULT 1,
first_occurred_at timestamptz NOT NULL,
last_occurred_at timestamptz NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE UNIQUE INDEX uq_business_alert_open
ON bitrix_sync.business_alerts(alert_type,fingerprint) WHERE status='open';
CREATE INDEX ix_business_alert_remote
ON bitrix_sync.business_alerts(remote_item_id) WHERE remote_item_id IS NOT NULL;
CREATE TABLE bitrix_sync.rebind_requests (
id uuid PRIMARY KEY,
user_id uuid NOT NULL,
old_external_id varchar(128),
target_external_id varchar(128) NOT NULL,
reason varchar(500) NOT NULL,
operator_id varchar(128) NOT NULL,
workflow_id uuid NOT NULL UNIQUE REFERENCES bitrix_sync.workflow_instances(id),
status varchar(24) NOT NULL CHECK (status IN
('pending','processing','retry_wait','succeeded','failed','cancelled')),
safe_error_code varchar(64),
requested_at timestamptz NOT NULL DEFAULT now(),
completed_at timestamptz,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX ix_rebind_claim
ON bitrix_sync.rebind_requests(status,requested_at)
WHERE status IN ('pending','retry_wait');
CREATE TABLE bitrix_sync.contact_snapshots (
id uuid PRIMARY KEY,
mapping_id uuid NOT NULL REFERENCES bitrix_sync.entity_external_mapping(id),
user_id uuid NOT NULL,
external_id varchar(128) NOT NULL,
full_name_hash varchar(64),
email_hash varchar(64),
phone_hash varchar(64),
citizenship_hash varchar(64),
citizenship_enum_id varchar(128),
citizenship_dictionary_loaded_at timestamptz,
source_updated_at timestamptz,
app_version varchar(128),
last_applied_source varchar(24) NOT NULL CHECK (last_applied_source IN
('webhook','reconciliation','app_create','app_update')),
last_webhook_received_at timestamptz,
last_webhook_source_at timestamptz,
profile_stale boolean NOT NULL DEFAULT false,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
UNIQUE(mapping_id)
);
CREATE INDEX ix_contact_snapshot_source
ON bitrix_sync.contact_snapshots(source_updated_at);
CREATE TABLE bitrix_sync.settings_versions (
id uuid PRIMARY KEY,
version bigint NOT NULL UNIQUE,
validation_status varchar(16) NOT NULL
CHECK (validation_status IN ('pending','valid','invalid')),
validation_errors jsonb NOT NULL DEFAULT '[]',
active boolean NOT NULL DEFAULT false,
created_by varchar(128) NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
activated_at timestamptz
);
CREATE UNIQUE INDEX uq_settings_version_active
ON bitrix_sync.settings_versions(active) WHERE active=true;
CREATE TABLE bitrix_sync.settings (
id uuid PRIMARY KEY,
version_id uuid NOT NULL REFERENCES bitrix_sync.settings_versions(id),
key varchar(128) NOT NULL,
value_type varchar(16) NOT NULL
CHECK (value_type IN ('integer','number','boolean','object')),
value_json jsonb NOT NULL,
validation_status varchar(16) NOT NULL CHECK (validation_status IN ('valid','invalid')),
active boolean NOT NULL DEFAULT false,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
UNIQUE(version_id,key)
);
CREATE INDEX ix_settings_active ON bitrix_sync.settings(key) WHERE active=true;
CREATE TABLE bitrix_sync.technical_dead_letters (
id uuid PRIMARY KEY,
workflow_id uuid REFERENCES bitrix_sync.workflow_instances(id),
command_id uuid REFERENCES bitrix_sync.crm_commands(id),
operation varchar(64) NOT NULL,
safe_error_code varchar(64) NOT NULL,
attempt_count integer NOT NULL DEFAULT 0,
deadline_at timestamptz,
correlation_id uuid,
failed_at timestamptz NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX ix_technical_dlq_failed
ON bitrix_sync.technical_dead_letters(failed_at DESC);
CREATE TABLE bitrix_sync.reconciliation_cursors (
job_type varchar(64) PRIMARY KEY,
watermark timestamptz NOT NULL,
overlap_seconds integer NOT NULL CHECK (overlap_seconds>=0),
last_success_at timestamptz,
last_scanned_count integer NOT NULL DEFAULT 0,
last_updated_count integer NOT NULL DEFAULT 0,
recovered_without_webhook_count integer NOT NULL DEFAULT 0,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE bitrix_sync.limiter_coordination (
limiter_key varchar(128) PRIMARY KEY,
tokens numeric(12,6) NOT NULL,
capacity numeric(12,6) NOT NULL CHECK (capacity>0),
refill_per_second numeric(12,6) NOT NULL CHECK (refill_per_second>0),
updated_at timestamptz NOT NULL,
blocked_until timestamptz,
method_class_blocks jsonb NOT NULL DEFAULT '{}',
fencing_token bigint NOT NULL DEFAULT 0
);
CREATE TABLE bitrix_sync.citizenship_dictionary (
enum_id varchar(128) PRIMARY KEY,
display_value varchar(255) NOT NULL,
loaded_at timestamptz NOT NULL,
expires_at timestamptz NOT NULL
);
"""
)
op.execute(
"""
CREATE OR REPLACE FUNCTION bitrix_sync.request_bitrix_contact_rebind(
p_user_id uuid,
p_target_b24_id varchar,
p_reason varchar,
p_operator_id varchar
) RETURNS uuid
LANGUAGE plpgsql
SECURITY DEFINER
SET search_path=bitrix_sync,pg_temp
AS $$
DECLARE
v_old_id varchar(128);
v_request_id uuid := gen_random_uuid();
v_workflow_id uuid := gen_random_uuid();
BEGIN
IF p_target_b24_id !~ '^[1-9][0-9]*$'
OR length(trim(p_reason)) < 5
OR length(trim(p_operator_id)) < 1 THEN
RAISE EXCEPTION 'invalid rebind request' USING ERRCODE='22023';
END IF;
SELECT external_id INTO v_old_id
FROM bitrix_sync.entity_external_mapping
WHERE entity_id=p_user_id AND entity_type='contact'
AND external_system='bitrix24' AND status='active'
FOR UPDATE;
IF EXISTS (
SELECT 1 FROM bitrix_sync.entity_external_mapping
WHERE external_system='bitrix24' AND external_entity_type='contact'
AND external_id=p_target_b24_id AND status='active'
AND entity_id<>p_user_id
) THEN
RAISE EXCEPTION 'target contact has another active mapping'
USING ERRCODE='23505';
END IF;
INSERT INTO bitrix_sync.workflow_instances
(id,workflow_type,user_id,external_id,state,current_step,deadline_at,created_at,updated_at)
VALUES
(v_workflow_id,'contact.rebind',p_user_id,p_target_b24_id,'created',
'validate_contacts',now()+interval '24 hours',now(),now());
INSERT INTO bitrix_sync.rebind_requests
(id,user_id,old_external_id,target_external_id,reason,operator_id,
workflow_id,status,requested_at,created_at,updated_at)
VALUES
(v_request_id,p_user_id,v_old_id,p_target_b24_id,trim(p_reason),
trim(p_operator_id),v_workflow_id,'pending',now(),now(),now());
RETURN v_request_id;
END;
$$;
"""
)
op.execute(
"""
REVOKE ALL ON FUNCTION
bitrix_sync.request_bitrix_contact_rebind(uuid,varchar,varchar,varchar)
FROM PUBLIC
"""
)
def downgrade() -> None:
raise RuntimeError("bitrix_sync production migration is forward-only")
@@ -0,0 +1,85 @@
"""Adopt the App queue contract and migrate legacy mappings.
Revision ID: 0002_app_queue_contract
Revises: 0001_bitrix_sync_full
Create Date: 2026-08-06
"""
from collections.abc import Sequence
from alembic import op
revision: str = "0002_app_queue_contract"
down_revision: str | None = "0001_bitrix_sync_full"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.execute(
"""
DO $$
BEGIN
IF to_regclass('han_app.entity_external_mapping') IS NOT NULL THEN
EXECUTE $copy$
INSERT INTO bitrix_sync.entity_external_mapping
(id,entity_type,entity_id,external_system,external_entity_type,
external_id,status,opened_at,closed_at,close_reason,created_at,updated_at)
SELECT id,entity_type,entity_id,'bitrix24','contact',
external_id,'active',coalesce(created_at,now()),NULL,NULL,
coalesce(created_at,now()),coalesce(created_at,now())
FROM han_app.entity_external_mapping
ON CONFLICT DO NOTHING
$copy$;
IF EXISTS (
SELECT 1 FROM han_app.entity_external_mapping old
LEFT JOIN bitrix_sync.entity_external_mapping new ON new.id=old.id
WHERE new.id IS NULL
) THEN
RAISE EXCEPTION 'legacy mapping migration verification failed';
END IF;
END IF;
END $$;
"""
)
op.execute(
"""
INSERT INTO bitrix_sync.settings_versions
(id,version,validation_status,active,created_by,created_at,activated_at)
VALUES ('00000000-0000-0000-0000-000000000001',1,'valid',true,'migration',now(),now())
ON CONFLICT DO NOTHING
"""
)
op.execute(
"""
INSERT INTO bitrix_sync.settings
(id,version_id,key,value_type,value_json,validation_status,active,created_at,updated_at)
VALUES
(gen_random_uuid(),'00000000-0000-0000-0000-000000000001','worker','object',
jsonb_build_object(
'batch_size',20,'batch_wait_ms',200,'claim_size',20,'lease_seconds',60,
'limiter_refill_per_sec',2,'limiter_burst',2,'max_in_flight',2,
'retry_base_seconds',1,'retry_max_seconds',900,'retry_horizon_seconds',86400
),
'valid',true,now(),now()),
(gen_random_uuid(),'00000000-0000-0000-0000-000000000001','reconciliation','object',
jsonb_build_object(
'contact_interval_seconds',900,'alert_interval_seconds',3600,
'overlap_seconds',300,'recovered_spike_threshold',20
),
'valid',true,now(),now()),
(gen_random_uuid(),'00000000-0000-0000-0000-000000000001','business_alerts','object',
jsonb_build_object(
'entity_type_id',NULL,'category_id',NULL,'stage_new',NULL,
'stage_in_progress',NULL,'stage_resolved',NULL,
'stage_closed_without_resolution',NULL,'sla_business_hours',8
),
'valid',true,now(),now())
ON CONFLICT DO NOTHING;
"""
)
def downgrade() -> None:
raise RuntimeError("App queue contract migration is forward-only")