mi
357 lines
15 KiB
Python
357 lines
15 KiB
Python
"""Create durable bitrix_sync schema and contracts.
|
|
|
|
Revision ID: 0001_bitrix_sync_full
|
|
Revises: 0001_sync_baseline
|
|
Create Date: 2026-08-06
|
|
"""
|
|
|
|
from collections.abc import Sequence
|
|
|
|
from alembic import op
|
|
|
|
revision: str = "0001_bitrix_sync_full"
|
|
down_revision: str | None = "0001_sync_baseline"
|
|
branch_labels: str | Sequence[str] | None = None
|
|
depends_on: str | Sequence[str] | None = None
|
|
|
|
|
|
def _execute_script(script: str) -> None:
|
|
"""Execute simple DDL statements separately for asyncpg compatibility."""
|
|
for statement in script.split(";"):
|
|
if statement.strip():
|
|
op.execute(statement)
|
|
|
|
|
|
def upgrade() -> None:
|
|
op.execute("CREATE EXTENSION IF NOT EXISTS pgcrypto")
|
|
op.execute("CREATE SCHEMA IF NOT EXISTS bitrix_sync")
|
|
_execute_script(
|
|
"""
|
|
CREATE TABLE bitrix_sync.workflow_instances (
|
|
id uuid PRIMARY KEY,
|
|
workflow_type varchar(64) NOT NULL CHECK (workflow_type IN
|
|
('contact.map_or_create','contact.update','contact.deactivate','contact.rebind',
|
|
'contact.webhook','contact.reconciliation','alert.reconciliation')),
|
|
user_id uuid,
|
|
external_id varchar(128),
|
|
state varchar(24) NOT NULL CHECK (state IN
|
|
('created','running','waiting_crm','waiting_retry','waiting_manual',
|
|
'succeeded','failed','cancelled')),
|
|
current_step varchar(64) NOT NULL,
|
|
source_task_id uuid UNIQUE,
|
|
deadline_at timestamptz NOT NULL,
|
|
outcome varchar(64),
|
|
completed_at timestamptz,
|
|
created_at timestamptz NOT NULL DEFAULT now(),
|
|
updated_at timestamptz NOT NULL DEFAULT now()
|
|
);
|
|
CREATE INDEX ix_workflow_claim
|
|
ON bitrix_sync.workflow_instances(state,updated_at)
|
|
WHERE state IN ('created','running','waiting_crm','waiting_retry');
|
|
|
|
CREATE TABLE bitrix_sync.entity_external_mapping (
|
|
id uuid PRIMARY KEY,
|
|
entity_type varchar(64) NOT NULL,
|
|
entity_id uuid NOT NULL,
|
|
external_system varchar(32) NOT NULL DEFAULT 'bitrix24'
|
|
CHECK (external_system='bitrix24'),
|
|
external_entity_type varchar(32) NOT NULL DEFAULT 'contact'
|
|
CHECK (external_entity_type='contact'),
|
|
external_id varchar(128) NOT NULL,
|
|
status varchar(16) NOT NULL CHECK (status IN ('active','closed','broken')),
|
|
opened_at timestamptz NOT NULL,
|
|
closed_at timestamptz,
|
|
close_reason varchar(64),
|
|
workflow_id uuid REFERENCES bitrix_sync.workflow_instances(id),
|
|
created_at timestamptz NOT NULL DEFAULT now(),
|
|
updated_at timestamptz NOT NULL DEFAULT now(),
|
|
CHECK ((status='active' AND closed_at IS NULL) OR
|
|
(status IN ('closed','broken')
|
|
AND (closed_at IS NOT NULL OR close_reason IS NOT NULL)))
|
|
);
|
|
CREATE UNIQUE INDEX uq_mapping_active_entity
|
|
ON bitrix_sync.entity_external_mapping(external_system,entity_type,entity_id)
|
|
WHERE status='active';
|
|
CREATE UNIQUE INDEX uq_mapping_active_external
|
|
ON bitrix_sync.entity_external_mapping
|
|
(external_system,external_entity_type,external_id)
|
|
WHERE status='active';
|
|
CREATE INDEX ix_mapping_history
|
|
ON bitrix_sync.entity_external_mapping(entity_id,opened_at DESC);
|
|
|
|
CREATE TABLE bitrix_sync.crm_commands (
|
|
id uuid PRIMARY KEY,
|
|
workflow_id uuid NOT NULL REFERENCES bitrix_sync.workflow_instances(id),
|
|
command_type varchar(40) NOT NULL CHECK (command_type IN
|
|
('duplicate_find','contact_get','contact_add','contact_update',
|
|
'citizenship_fields_get','contact_incremental_list',
|
|
'alert_get','alert_add','alert_update',
|
|
'rebind_target_get','rebind_old_get')),
|
|
safe_request jsonb NOT NULL DEFAULT '{}',
|
|
status varchar(24) NOT NULL CHECK (status IN
|
|
('pending','leased','in_flight','succeeded','retry','retry_wait',
|
|
'uncertain','reconcile','dead_letter','permanent','rate_limited')),
|
|
attempt_count integer NOT NULL DEFAULT 0 CHECK (attempt_count>=0),
|
|
next_attempt_at timestamptz NOT NULL DEFAULT now(),
|
|
locked_by varchar(128),
|
|
locked_until timestamptz,
|
|
lease_token uuid,
|
|
batch_id uuid,
|
|
correlation_id uuid,
|
|
safe_response jsonb,
|
|
safe_error_code varchar(64),
|
|
http_status integer,
|
|
completed_at timestamptz,
|
|
created_at timestamptz NOT NULL DEFAULT now(),
|
|
updated_at timestamptz NOT NULL DEFAULT now()
|
|
);
|
|
CREATE INDEX ix_crm_command_claim
|
|
ON bitrix_sync.crm_commands(status,next_attempt_at,created_at);
|
|
CREATE INDEX ix_crm_command_workflow
|
|
ON bitrix_sync.crm_commands(workflow_id,created_at);
|
|
|
|
CREATE TABLE bitrix_sync.webhook_inbox (
|
|
id uuid PRIMARY KEY,
|
|
receiver_type varchar(16) NOT NULL CHECK (receiver_type IN ('contact','alert')),
|
|
event_type varchar(64) NOT NULL,
|
|
event_id varchar(255),
|
|
source_timestamp timestamptz,
|
|
external_entity_id varchar(128) NOT NULL,
|
|
dedup_fingerprint varchar(64),
|
|
source_ip inet,
|
|
status varchar(16) NOT NULL CHECK (status IN
|
|
('received','coalesced','processing','processed','retry_wait','dead_letter')),
|
|
coalesced_count integer NOT NULL DEFAULT 1,
|
|
attempt_count integer NOT NULL DEFAULT 0,
|
|
next_attempt_at timestamptz NOT NULL DEFAULT now(),
|
|
locked_by varchar(128),
|
|
locked_until timestamptz,
|
|
lease_token uuid,
|
|
received_at timestamptz NOT NULL,
|
|
last_received_at timestamptz NOT NULL,
|
|
processed_at timestamptz,
|
|
safe_error_code varchar(64)
|
|
);
|
|
CREATE UNIQUE INDEX uq_webhook_event_id
|
|
ON bitrix_sync.webhook_inbox(receiver_type,event_id) WHERE event_id IS NOT NULL;
|
|
CREATE INDEX ix_webhook_claim
|
|
ON bitrix_sync.webhook_inbox(status,next_attempt_at,received_at);
|
|
CREATE UNIQUE INDEX uq_webhook_active_entity
|
|
ON bitrix_sync.webhook_inbox(receiver_type,external_entity_id)
|
|
WHERE status IN ('received','processing','retry_wait');
|
|
"""
|
|
)
|
|
_execute_script(
|
|
"""
|
|
CREATE SEQUENCE bitrix_sync.business_alert_number_seq;
|
|
CREATE TABLE bitrix_sync.business_alerts (
|
|
id uuid PRIMARY KEY,
|
|
alert_number bigint NOT NULL DEFAULT nextval('bitrix_sync.business_alert_number_seq'),
|
|
fingerprint varchar(64) NOT NULL,
|
|
alert_type varchar(64) NOT NULL,
|
|
severity varchar(16) NOT NULL CHECK (severity IN ('info','warning','critical')),
|
|
app_user_id uuid,
|
|
current_external_id varchar(128),
|
|
selected_external_id varchar(128),
|
|
candidate_external_ids text[] NOT NULL DEFAULT '{}',
|
|
remote_item_id varchar(128),
|
|
remote_stage_id varchar(128),
|
|
previous_alert_id uuid REFERENCES bitrix_sync.business_alerts(id),
|
|
workflow_id uuid REFERENCES bitrix_sync.workflow_instances(id),
|
|
status varchar(24) NOT NULL CHECK (status IN
|
|
('open','in_progress','resolved','closed_without_resolution','remote_missing')),
|
|
occurrence_count integer NOT NULL DEFAULT 1,
|
|
first_occurred_at timestamptz NOT NULL,
|
|
last_occurred_at timestamptz NOT NULL,
|
|
created_at timestamptz NOT NULL DEFAULT now(),
|
|
updated_at timestamptz NOT NULL DEFAULT now()
|
|
);
|
|
CREATE UNIQUE INDEX uq_business_alert_open
|
|
ON bitrix_sync.business_alerts(alert_type,fingerprint) WHERE status='open';
|
|
CREATE INDEX ix_business_alert_remote
|
|
ON bitrix_sync.business_alerts(remote_item_id) WHERE remote_item_id IS NOT NULL;
|
|
|
|
CREATE TABLE bitrix_sync.rebind_requests (
|
|
id uuid PRIMARY KEY,
|
|
user_id uuid NOT NULL,
|
|
old_external_id varchar(128),
|
|
target_external_id varchar(128) NOT NULL,
|
|
reason varchar(500) NOT NULL,
|
|
operator_id varchar(128) NOT NULL,
|
|
workflow_id uuid NOT NULL UNIQUE REFERENCES bitrix_sync.workflow_instances(id),
|
|
status varchar(24) NOT NULL CHECK (status IN
|
|
('pending','processing','retry_wait','succeeded','failed','cancelled')),
|
|
safe_error_code varchar(64),
|
|
requested_at timestamptz NOT NULL DEFAULT now(),
|
|
completed_at timestamptz,
|
|
created_at timestamptz NOT NULL DEFAULT now(),
|
|
updated_at timestamptz NOT NULL DEFAULT now()
|
|
);
|
|
CREATE INDEX ix_rebind_claim
|
|
ON bitrix_sync.rebind_requests(status,requested_at)
|
|
WHERE status IN ('pending','retry_wait');
|
|
|
|
CREATE TABLE bitrix_sync.contact_snapshots (
|
|
id uuid PRIMARY KEY,
|
|
mapping_id uuid NOT NULL REFERENCES bitrix_sync.entity_external_mapping(id),
|
|
user_id uuid NOT NULL,
|
|
external_id varchar(128) NOT NULL,
|
|
full_name_hash varchar(64),
|
|
email_hash varchar(64),
|
|
phone_hash varchar(64),
|
|
citizenship_hash varchar(64),
|
|
citizenship_enum_id varchar(128),
|
|
citizenship_dictionary_loaded_at timestamptz,
|
|
source_updated_at timestamptz,
|
|
app_version varchar(128),
|
|
last_applied_source varchar(24) NOT NULL CHECK (last_applied_source IN
|
|
('webhook','reconciliation','app_create','app_update')),
|
|
last_webhook_received_at timestamptz,
|
|
last_webhook_source_at timestamptz,
|
|
profile_stale boolean NOT NULL DEFAULT false,
|
|
created_at timestamptz NOT NULL DEFAULT now(),
|
|
updated_at timestamptz NOT NULL DEFAULT now(),
|
|
UNIQUE(mapping_id)
|
|
);
|
|
CREATE INDEX ix_contact_snapshot_source
|
|
ON bitrix_sync.contact_snapshots(source_updated_at);
|
|
|
|
CREATE TABLE bitrix_sync.settings_versions (
|
|
id uuid PRIMARY KEY,
|
|
version bigint NOT NULL UNIQUE,
|
|
validation_status varchar(16) NOT NULL
|
|
CHECK (validation_status IN ('pending','valid','invalid')),
|
|
validation_errors jsonb NOT NULL DEFAULT '[]',
|
|
active boolean NOT NULL DEFAULT false,
|
|
created_by varchar(128) NOT NULL,
|
|
created_at timestamptz NOT NULL DEFAULT now(),
|
|
activated_at timestamptz
|
|
);
|
|
CREATE UNIQUE INDEX uq_settings_version_active
|
|
ON bitrix_sync.settings_versions(active) WHERE active=true;
|
|
|
|
CREATE TABLE bitrix_sync.settings (
|
|
id uuid PRIMARY KEY,
|
|
version_id uuid NOT NULL REFERENCES bitrix_sync.settings_versions(id),
|
|
key varchar(128) NOT NULL,
|
|
value_type varchar(16) NOT NULL
|
|
CHECK (value_type IN ('integer','number','boolean','object')),
|
|
value_json jsonb NOT NULL,
|
|
validation_status varchar(16) NOT NULL CHECK (validation_status IN ('valid','invalid')),
|
|
active boolean NOT NULL DEFAULT false,
|
|
created_at timestamptz NOT NULL DEFAULT now(),
|
|
updated_at timestamptz NOT NULL DEFAULT now(),
|
|
UNIQUE(version_id,key)
|
|
);
|
|
CREATE INDEX ix_settings_active ON bitrix_sync.settings(key) WHERE active=true;
|
|
|
|
CREATE TABLE bitrix_sync.technical_dead_letters (
|
|
id uuid PRIMARY KEY,
|
|
workflow_id uuid REFERENCES bitrix_sync.workflow_instances(id),
|
|
command_id uuid REFERENCES bitrix_sync.crm_commands(id),
|
|
operation varchar(64) NOT NULL,
|
|
safe_error_code varchar(64) NOT NULL,
|
|
attempt_count integer NOT NULL DEFAULT 0,
|
|
deadline_at timestamptz,
|
|
correlation_id uuid,
|
|
failed_at timestamptz NOT NULL,
|
|
created_at timestamptz NOT NULL DEFAULT now()
|
|
);
|
|
CREATE INDEX ix_technical_dlq_failed
|
|
ON bitrix_sync.technical_dead_letters(failed_at DESC);
|
|
|
|
CREATE TABLE bitrix_sync.reconciliation_cursors (
|
|
job_type varchar(64) PRIMARY KEY,
|
|
watermark timestamptz NOT NULL,
|
|
overlap_seconds integer NOT NULL CHECK (overlap_seconds>=0),
|
|
last_success_at timestamptz,
|
|
last_scanned_count integer NOT NULL DEFAULT 0,
|
|
last_updated_count integer NOT NULL DEFAULT 0,
|
|
recovered_without_webhook_count integer NOT NULL DEFAULT 0,
|
|
created_at timestamptz NOT NULL DEFAULT now(),
|
|
updated_at timestamptz NOT NULL DEFAULT now()
|
|
);
|
|
|
|
CREATE TABLE bitrix_sync.limiter_coordination (
|
|
limiter_key varchar(128) PRIMARY KEY,
|
|
tokens numeric(12,6) NOT NULL,
|
|
capacity numeric(12,6) NOT NULL CHECK (capacity>0),
|
|
refill_per_second numeric(12,6) NOT NULL CHECK (refill_per_second>0),
|
|
updated_at timestamptz NOT NULL,
|
|
blocked_until timestamptz,
|
|
method_class_blocks jsonb NOT NULL DEFAULT '{}',
|
|
fencing_token bigint NOT NULL DEFAULT 0
|
|
);
|
|
|
|
CREATE TABLE bitrix_sync.citizenship_dictionary (
|
|
enum_id varchar(128) PRIMARY KEY,
|
|
display_value varchar(255) NOT NULL,
|
|
loaded_at timestamptz NOT NULL,
|
|
expires_at timestamptz NOT NULL
|
|
);
|
|
"""
|
|
)
|
|
op.execute(
|
|
"""
|
|
CREATE OR REPLACE FUNCTION bitrix_sync.request_bitrix_contact_rebind(
|
|
p_user_id uuid,
|
|
p_target_b24_id varchar,
|
|
p_reason varchar,
|
|
p_operator_id varchar
|
|
) RETURNS uuid
|
|
LANGUAGE plpgsql
|
|
SECURITY DEFINER
|
|
SET search_path=bitrix_sync,pg_temp
|
|
AS $$
|
|
DECLARE
|
|
v_old_id varchar(128);
|
|
v_request_id uuid := gen_random_uuid();
|
|
v_workflow_id uuid := gen_random_uuid();
|
|
BEGIN
|
|
IF p_target_b24_id !~ '^[1-9][0-9]*$'
|
|
OR length(trim(p_reason)) < 5
|
|
OR length(trim(p_operator_id)) < 1 THEN
|
|
RAISE EXCEPTION 'invalid rebind request' USING ERRCODE='22023';
|
|
END IF;
|
|
SELECT external_id INTO v_old_id
|
|
FROM bitrix_sync.entity_external_mapping
|
|
WHERE entity_id=p_user_id AND entity_type='contact'
|
|
AND external_system='bitrix24' AND status='active'
|
|
FOR UPDATE;
|
|
IF EXISTS (
|
|
SELECT 1 FROM bitrix_sync.entity_external_mapping
|
|
WHERE external_system='bitrix24' AND external_entity_type='contact'
|
|
AND external_id=p_target_b24_id AND status='active'
|
|
AND entity_id<>p_user_id
|
|
) THEN
|
|
RAISE EXCEPTION 'target contact has another active mapping'
|
|
USING ERRCODE='23505';
|
|
END IF;
|
|
INSERT INTO bitrix_sync.workflow_instances
|
|
(id,workflow_type,user_id,external_id,state,current_step,deadline_at,created_at,updated_at)
|
|
VALUES
|
|
(v_workflow_id,'contact.rebind',p_user_id,p_target_b24_id,'created',
|
|
'validate_contacts',now()+interval '24 hours',now(),now());
|
|
INSERT INTO bitrix_sync.rebind_requests
|
|
(id,user_id,old_external_id,target_external_id,reason,operator_id,
|
|
workflow_id,status,requested_at,created_at,updated_at)
|
|
VALUES
|
|
(v_request_id,p_user_id,v_old_id,p_target_b24_id,trim(p_reason),
|
|
trim(p_operator_id),v_workflow_id,'pending',now(),now(),now());
|
|
RETURN v_request_id;
|
|
END;
|
|
$$;
|
|
"""
|
|
)
|
|
op.execute(
|
|
"""
|
|
REVOKE ALL ON FUNCTION
|
|
bitrix_sync.request_bitrix_contact_rebind(uuid,varchar,varchar,varchar)
|
|
FROM PUBLIC
|
|
"""
|
|
)
|
|
|
|
|
|
def downgrade() -> None:
|
|
raise RuntimeError("bitrix_sync production migration is forward-only")
|