commit b74b024b6cf7ba6a9c112580523c4e430cce23a2 · zenfyr.dev/xpost

+101

atproto/identity.py

···

       1
       1
       +
       from typing import Any

     

       2
       2
       +
       import dns.resolver

     

       3
       3
       +
       import requests

     

       4
       4
       +
       from util.cache import TTLCache

     

       5
       5
       +
       from util.util import LOGGER

     

       6
       6
       +
       

     

       7
       7
       +
       class DidResolver:

     

       8
       8
       +
           def __init__(self, plc_host: str) -> None:

     

       9
       9
       +
               self.plc_host: str = plc_host

     

       10
       10
       +
               self.__cache: TTLCache[str, dict[str, Any]] = TTLCache(ttl_seconds=12*60*60)

     

       11
       11
       +
       

     

       12
       12
       +
           def try_resolve_plc(self, did: str) -> dict[str, Any] | None:

     

       13
       13
       +
               url = f"{self.plc_host}/{did}"

     

       14
       14
       +
               response = requests.get(url, timeout=10, allow_redirects=True)

     

       15
       15
       +
       

     

       16
       16
       +
               if response.status_code == 200:

     

       17
       17
       +
                   return response.json()

     

       18
       18
       +
               elif response.status_code == 404 or response.status_code == 410:

     

       19
       19
       +
                   return None # tombstone or not registered

     

       20
       20
       +
               else:

     

       21
       21
       +
                   response.raise_for_status()

     

       22
       22
       +
       

     

       23
       23
       +
           def try_resolve_web(self, did: str) -> dict[str, Any] | None:

     

       24
       24
       +
               url = f"http://{did[len('did:web:'):]}/.well-known/did.json"

     

       25
       25
       +
               response = requests.get(url, timeout=10, allow_redirects=True)

     

       26
       26
       +
       

     

       27
       27
       +
               if response.status_code == 200:

     

       28
       28
       +
                   return response.json()

     

       29
       29
       +
               elif response.status_code == 404 or response.status_code == 410:

     

       30
       30
       +
                   return None # tombstone or gone

     

       31
       31
       +
               else:

     

       32
       32
       +
                   response.raise_for_status()

     

       33
       33
       +
       

     

       34
       34
       +
           def resolve_did(self, did: str) -> dict[str, Any]:

     

       35
       35
       +
               cached = self.__cache.get(did)

     

       36
       36
       +
               if cached:

     

       37
       37
       +
                   return cached

     

       38
       38
       +
       

     

       39
       39
       +
               if did.startswith('did:plc:'):

     

       40
       40
       +
                   from_plc = self.try_resolve_plc(did)

     

       41
       41
       +
                   if from_plc:

     

       42
       42
       +
                       self.__cache.set(did, from_plc)

     

       43
       43
       +
                       return from_plc

     

       44
       44
       +
               elif did.startswith('did:web:'):

     

       45
       45
       +
                   from_web = self.try_resolve_web(did)

     

       46
       46
       +
                   if from_web:

     

       47
       47
       +
                       self.__cache.set(did, from_web)

     

       48
       48
       +
                       return from_web

     

       49
       49
       +
               raise Exception(f"Failed to resolve {did}!")

     

       50
       50
       +
       

     

       51
       51
       +
       class HandleResolver:

     

       52
       52
       +
           def __init__(self) -> None:

     

       53
       53
       +
               self.__cache: TTLCache[str, str] = TTLCache()

     

       54
       54
       +
       

     

       55
       55
       +
           def try_resolve_dns(self, handle: str) -> str | None:

     

       56
       56
       +
               try:

     

       57
       57
       +
                   dns_query = f"_atproto.{handle}"

     

       58
       58
       +
                   answers = dns.resolver.resolve(dns_query, "TXT")

     

       59
       59
       +
       

     

       60
       60
       +
                   for rdata in answers:

     

       61
       61
       +
                       for txt_data in rdata.strings:

     

       62
       62
       +
                           did = txt_data.decode('utf-8').strip()

     

       63
       63
       +
                           if did.startswith("did="):

     

       64
       64
       +
                               return did[4:]

     

       65
       65
       +
               except dns.resolver.NXDOMAIN:

     

       66
       66
       +
                   LOGGER.debug(f"DNS record not found for _atproto.{handle}")

     

       67
       67
       +
                   return None

     

       68
       68
       +
               except dns.resolver.NoAnswer:

     

       69
       69
       +
                   LOGGER.debug(f"No TXT records found for _atproto.{handle}")

     

       70
       70
       +
                   return None

     

       71
       71
       +
       

     

       72
       72
       +
           def try_resolve_http(self, handle: str) -> str | None:

     

       73
       73
       +
               url = f"http://{handle}/.well-known/atproto-did"

     

       74
       74
       +
               response = requests.get(url, timeout=10, allow_redirects=True)

     

       75
       75
       +
       

     

       76
       76
       +
               if response.status_code == 200:

     

       77
       77
       +
                   did = response.text.strip()

     

       78
       78
       +
                   if did.startswith("did:"):

     

       79
       79
       +
                       return did

     

       80
       80
       +
                   else:

     

       81
       81
       +
                       raise ValueError(f"Got invalid did: from {url} = {did}!")

     

       82
       82
       +
               else:

     

       83
       83
       +
                   response.raise_for_status()

     

       84
       84
       +
       

     

       85
       85
       +
       

     

       86
       86
       +
           def resolve_handle(self, handle: str) -> str:

     

       87
       87
       +
               cached = self.__cache.get(handle)

     

       88
       88
       +
               if cached:

     

       89
       89
       +
                   return cached

     

       90
       90
       +
       

     

       91
       91
       +
               from_dns = self.try_resolve_dns(handle)

     

       92
       92
       +
               if from_dns:

     

       93
       93
       +
                   self.__cache.set(handle, from_dns)

     

       94
       94
       +
                   return from_dns

     

       95
       95
       +
       

     

       96
       96
       +
               from_http = self.try_resolve_http(handle)

     

       97
       97
       +
               if from_http:

     

       98
       98
       +
                   self.__cache.set(handle, from_http)

     

       99
       99
       +
                   return from_http

     

       100
       100
       +
       

     

       101
       101
       +
               raise Exception(f"Failed to resolve handle {handle}!")

+19

bluesky/info.py

···

       1
       1
       +
       from abc import ABC

     

       2
       2
       +
       from typing import Any

     

       3
       3
       +
       from cross.service import Service

     

       4
       4
       +
       from util.util import normalize_service_url

     

       5
       5
       +
       

     

       6
       6
       +
       def validate_and_transform(data: dict[str, Any]):

     

       7
       7
       +
           if not data["handle"] and not data["did"]:

     

       8
       8
       +
               raise KeyError("no 'handle' or 'did' specified for bluesky input!")

     

       9
       9
       +
       

     

       10
       10
       +
           if "did" in data:

     

       11
       11
       +
               did = str(data["did"])  # only did:web and did:plc are supported

     

       12
       12
       +
               if not did.startswith("did:plc:") and not did.startswith("did:web:"):

     

       13
       13
       +
                   raise ValueError(f"Invalid handle {did}!")

     

       14
       14
       +
       

     

       15
       15
       +
           if "pds" in data:

     

       16
       16
       +
               data["pds"] = normalize_service_url(data["pds"])

     

       17
       17
       +
       

     

       18
       18
       +
       class BlueskyService(ABC, Service):

     

       19
       19
       +
           pass

+38

bluesky/input.py

···

       1
       1
       +
       from abc import ABC

     

       2
       2
       +
       from dataclasses import dataclass, field

     

       3
       3
       +
       import re

     

       4
       4
       +
       from typing import Any, Callable, override

     

       5
       5
       +
       

     

       6
       6
       +
       from bluesky.info import BlueskyService, validate_and_transform

     

       7
       7
       +
       from cross.service import InputService, OutputService

     

       8
       8
       +
       

     

       9
       9
       +
       

     

       10
       10
       +
       @dataclass(kw_only=True)

     

       11
       11
       +
       class BlueskyInputOptions:

     

       12
       12
       +
           handle: str | None

     

       13
       13
       +
           did: str | None

     

       14
       14
       +
           pds: str | None

     

       15
       15
       +
           filters: list[re.Pattern[str]] = field(default_factory=lambda: [])

     

       16
       16
       +
       

     

       17
       17
       +
           @classmethod

     

       18
       18
       +
           def from_dict(cls, data: dict[str, Any]) -> "BlueskyInputOptions":

     

       19
       19
       +
               validate_and_transform(data)

     

       20
       20
       +
       

     

       21
       21
       +
               if "filters" in data:

     

       22
       22
       +
                   data["filters"] = [re.compile(r) for r in data["filters"]]

     

       23
       23
       +
       

     

       24
       24
       +
               return BlueskyInputOptions(**data)

     

       25
       25
       +
       

     

       26
       26
       +
       

     

       27
       27
       +
       class BlueskyBaseInputService(BlueskyService, InputService, ABC):

     

       28
       28
       +
           pass

     

       29
       29
       +
       

     

       30
       30
       +
       

     

       31
       31
       +
       class BlueskyJetstreamInputService(BlueskyBaseInputService):

     

       32
       32
       +
           @override

     

       33
       33
       +
           async def listen(

     

       34
       34
       +
               self,

     

       35
       35
       +
               outputs: list[OutputService],

     

       36
       36
       +
               submitter: Callable[[Callable[[], None]], None],

     

       37
       37
       +
           ):

     

       38
       38
       +
               return await super().listen(outputs, submitter)  # TODO

+4 -2

cross/service.py

···

       1
       1
        
       import sqlite3

     

       2
       2
       +
       from abc import ABC, abstractmethod

     

       2
       3
        
       from pathlib import Path

     

       3
       4
        
       from typing import Callable, cast

     

       4
       5
        
       

     
···

       53
       54
        
               LOGGER.warning("NOT IMPLEMENTED (%s), delete_repost %s", self.url, repost_id)

     

       54
       55
        
       

     

       55
       56
        
       

     

       56
       56
       -
       class InputService(Service):

     

       57
       57
       +
       class InputService(ABC, Service):

     

       58
       58
       +
           @abstractmethod

     

       57
       59
        
           async def listen(

     

       58
       60
        
               self,

     

       59
       61
        
               outputs: list[OutputService],

     

       60
       62
        
               submitter: Callable[[Callable[[], None]], None],

     

       61
       61
       -
           ):  # pyright: ignore[reportUnusedParameter]

     

       63
       63
       +
           ):

     

       62
       64
        
               pass

env.py

···

       1
       1
        
       import os

     

       2
       2
        
       

     

       3
       3
       +
       DEV = bool(os.environ.get("DEV")) or False

     

       3
       4
        
       DATA_DIR = os.environ.get("DATA_DIR") or "./data"

     

       4
       5
        
       MIGRATIONS_DIR = os.environ.get("MIGRATIONS_DIR") or "./migrations"

     

       6
       6
       +
       PLC_HOST = os.environ.get("PLC_HOST") or "http://plc.directory"

+6 -1

mastodon/info.py

···

       5
       5
        
       import requests

     

       6
       6
        
       

     

       7
       7
        
       from cross.service import Service

     

       8
       8
       -
       from util.util import LOGGER

     

       8
       8
       +
       from util.util import LOGGER, normalize_service_url

     

       9
       9
       +
       

     

       10
       10
       +
       def validate_and_transform(data: dict[str, Any]):

     

       11
       11
       +
           if 'token' not in data or 'instance' not in data:

     

       12
       12
       +
               raise KeyError("Missing required values 'token' or 'instance'")

     

       9
       13
        
       

     

       14
       14
       +
           data["instance"] = normalize_service_url(data["instance"])

     

       10
       15
        
       

     

       11
       16
        
       @dataclass(kw_only=True)

     

       12
       17
        
       class InstanceInfo:

+5 -9

mastodon/input.py

···

       7
       7
        
       import websockets

     

       8
       8
        
       

     

       9
       9
        
       from cross.service import InputService, OutputService

     

       10
       10
       -
       from mastodon.info import MastodonService

     

       10
       10
       +
       from mastodon.info import MastodonService, validate_and_transform

     

       11
       11
        
       from util.util import LOGGER

     

       12
       12
        
       

     

       13
       13
        
       ALLOWED_VISIBILITY: list[str] = ["public", "unlisted"]

     
···

       20
       20
        
           allowed_visibility: list[str] = field(

     

       21
       21
        
               default_factory=lambda: ALLOWED_VISIBILITY.copy()

     

       22
       22
        
           )

     

       23
       23
       -
           regex_filters: list[re.Pattern[str]] = field(default_factory=lambda: [])

     

       23
       23
       +
           filters: list[re.Pattern[str]] = field(default_factory=lambda: [])

     

       24
       24
        
       

     

       25
       25
        
           @classmethod

     

       26
       26
        
           def from_dict(cls, data: dict[str, Any]) -> "MastodonInputOptions":

     

       27
       27
       -
               data["instance"] = (

     

       28
       28
       -
                   data["instance"][:-1]

     

       29
       29
       -
                   if data["instance"].endswith("/")

     

       30
       30
       -
                   else data["instance"]

     

       31
       31
       -
               )

     

       27
       27
       +
               validate_and_transform(data)

     

       32
       28
        
       

     

       33
       29
        
               if "allowed_visibility" in data:

     

       34
       30
        
                   for vis in data.get("allowed_visibility", []):

     

       35
       31
        
                       if vis not in ALLOWED_VISIBILITY:

     

       36
       32
        
                           raise ValueError(f"Invalid visibility option {vis}!")

     

       37
       33
        
       

     

       38
       38
       -
               if "regex_filters" in data:

     

       39
       39
       -
                   data["regex_filters"] = [re.compile(r) for r in data["regex_filters"]]

     

       34
       34
       +
               if "filters" in data:

     

       35
       35
       +
                   data["filters"] = [re.compile(r) for r in data["filters"]]

     

       40
       36
        
       

     

       41
       37
        
               return MastodonInputOptions(**data)

     

       42
       38

+3 -7

mastodon/output.py

···

       3
       3
        
       from typing import Any, override

     

       4
       4
        
       

     

       5
       5
        
       from cross.service import OutputService

     

       6
       6
       -
       from mastodon.info import InstanceInfo, MastodonService

     

       6
       6
       +
       from mastodon.info import InstanceInfo, MastodonService, validate_and_transform

     

       7
       7
        
       from util.util import LOGGER

     

       8
       8
        
       

     

       9
       9
        
       ALLOWED_POSTING_VISIBILITY: list[str] = ["public", "unlisted", "private"]

     
···

       17
       17
        
       

     

       18
       18
        
           @classmethod

     

       19
       19
        
           def from_dict(cls, data: dict[str, Any]) -> "MastodonOutputOptions":

     

       20
       20
       -
               data["instance"] = (

     

       21
       21
       -
                   data["instance"][:-1]

     

       22
       22
       -
                   if data["instance"].endswith("/")

     

       23
       23
       -
                   else data["instance"]

     

       24
       24
       -
               )

     

       20
       20
       +
               validate_and_transform(data)

     

       25
       21
        
       

     

       26
       22
        
               if "visibility" in data:

     

       27
       23
        
                   if data["visibility"] not in ALLOWED_POSTING_VISIBILITY:

     
···

       42
       38
        
       

     

       43
       39
        
               LOGGER.info("Getting %s configuration...", self.url)

     

       44
       40
        
               responce = self.fetch_instance_info()

     

       45
       45
       -
               self.instance_info = InstanceInfo.from_api(responce)

     

       41
       41
       +
               self.instance_info: InstanceInfo = InstanceInfo.from_api(responce)

     

       46
       42
        
       

     

       47
       43
        
           @override

     

       48
       44
        
           def _get_token(self) -> str:

+5 -9

misskey/input.py

···

       10
       10
        
       

     

       11
       11
        
       from cross.service import InputService, OutputService

     

       12
       12
        
       from misskey.info import MisskeyService

     

       13
       13
       -
       from util.util import LOGGER

     

       13
       13
       +
       from util.util import LOGGER, normalize_service_url

     

       14
       14
        
       

     

       15
       15
        
       ALLOWED_VISIBILITY = ["public", "home"]

     

       16
       16
        
       

     
···

       22
       22
        
           allowed_visibility: list[str] = field(

     

       23
       23
        
               default_factory=lambda: ALLOWED_VISIBILITY.copy()

     

       24
       24
        
           )

     

       25
       25
       -
           regex_filters: list[re.Pattern[str]] = field(default_factory=lambda: [])

     

       25
       25
       +
           filters: list[re.Pattern[str]] = field(default_factory=lambda: [])

     

       26
       26
        
       

     

       27
       27
        
           @classmethod

     

       28
       28
        
           def from_dict(cls, data: dict[str, Any]) -> "MisskeyInputOptions":

     

       29
       29
       -
               data["instance"] = (

     

       30
       30
       -
                   data["instance"][:-1]

     

       31
       31
       -
                   if data["instance"].endswith("/")

     

       32
       32
       -
                   else data["instance"]

     

       33
       33
       -
               )

     

       29
       29
       +
               data["instance"] = normalize_service_url(data["instance"])

     

       34
       30
        
       

     

       35
       31
        
               if "allowed_visibility" in data:

     

       36
       32
        
                   for vis in data.get("allowed_visibility", []):

     

       37
       33
        
                       if vis not in ALLOWED_VISIBILITY:

     

       38
       34
        
                           raise ValueError(f"Invalid visibility option {vis}!")

     

       39
       35
        
       

     

       40
       40
       -
               if "regex_filters" in data:

     

       41
       41
       -
                   data["regex_filters"] = [re.compile(r) for r in data["regex_filters"]]

     

       36
       36
       +
               if "filters" in data:

     

       37
       37
       +
                   data["filters"] = [re.compile(r) for r in data["filters"]]

     

       42
       38
        
       

     

       43
       39
        
               return MisskeyInputOptions(**data)

     

       44
       40

pyproject.toml

···

       5
       5
        
       readme = "README.md"

     

       6
       6
        
       requires-python = ">=3.12"

     

       7
       7
        
       dependencies = [

     

       8
       8
       +
           "dnspython>=2.8.0",

     

       8
       9
        
           "python-magic>=0.4.27",

     

       9
       10
        
           "requests>=2.32.5",

     

       10
       11
        
           "websockets>=15.0.1",

+25

util/cache.py

···

       1
       1
       +
       import time

     

       2
       2
       +
       from typing import Generic, TypeVar

     

       3
       3
       +
       

     

       4
       4
       +
       K = TypeVar("K")

     

       5
       5
       +
       V = TypeVar("V")

     

       6
       6
       +
       

     

       7
       7
       +
       class TTLCache(Generic[K, V]):

     

       8
       8
       +
           def __init__(self, ttl_seconds: int = 3600) -> None:

     

       9
       9
       +
               self.ttl: int = ttl_seconds

     

       10
       10
       +
               self.__cache: dict[K, tuple[V, float]] = {}

     

       11
       11
       +
       

     

       12
       12
       +
           def get(self, key: K) -> V | None:

     

       13
       13
       +
               if key in self.__cache:

     

       14
       14
       +
                   value, timestamp = self.__cache[key]

     

       15
       15
       +
                   if time.time() - timestamp < self.ttl:

     

       16
       16
       +
                       return value

     

       17
       17
       +
                   else:

     

       18
       18
       +
                       del self.__cache[key]

     

       19
       19
       +
               return None

     

       20
       20
       +
       

     

       21
       21
       +
           def set(self, key: K, value: V) -> None:

     

       22
       22
       +
                   self.__cache[key] = (value, time.time())

     

       23
       23
       +
       

     

       24
       24
       +
           def clear(self) -> None:

     

       25
       25
       +
               self.__cache.clear()

+9 -1

util/util.py

···

       3
       3
        
       import os

     

       4
       4
        
       from typing import Any

     

       5
       5
        
       

     

       6
       6
       -
       logging.basicConfig(stream=sys.stderr, level=logging.INFO)

     

       6
       6
       +
       import env

     

       7
       7
       +
       

     

       8
       8
       +
       logging.basicConfig(stream=sys.stderr, level=logging.DEBUG if env.DEV else logging.INFO)

     

       7
       9
        
       LOGGER = logging.getLogger("XPost")

     

       10
       10
       +
       

     

       11
       11
       +
       def normalize_service_url(url: str) -> str:

     

       12
       12
       +
           if not url.startswith("https://") and not url.startswith("http://"):

     

       13
       13
       +
               raise ValueError(f"Invalid service url {url}! Only http/https are supported.")

     

       14
       14
       +
       

     

       15
       15
       +
           return url[:-1] if url.endswith('/') else url

     

       8
       16
        
       

     

       9
       17
        
       def read_env(data: dict[str, Any]) -> None:

     

       10
       18
        
           keys = list(data.keys())

+11

uv.lock

···

       69
       69
        
       ]

     

       70
       70
        
       

     

       71
       71
        
       [[package]]

     

       72
       72
       +
       name = "dnspython"

     

       73
       73
       +
       version = "2.8.0"

     

       74
       74
       +
       source = { registry = "https://pypi.org/simple" }

     

       75
       75
       +
       sdist = { url = "https://files.pythonhosted.org/packages/8c/8b/57666417c0f90f08bcafa776861060426765fdb422eb10212086fb811d26/dnspython-2.8.0.tar.gz", hash = "sha256:181d3c6996452cb1189c4046c61599b84a5a86e099562ffde77d26984ff26d0f", size = 368251, upload-time = "2025-09-07T18:58:00.022Z" }

     

       76
       76
       +
       wheels = [

     

       77
       77
       +
           { url = "https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", size = 331094, upload-time = "2025-09-07T18:57:58.071Z" },

     

       78
       78
       +
       ]

     

       79
       79
       +
       

     

       80
       80
       +
       [[package]]

     

       72
       81
        
       name = "idna"

     

       73
       82
        
       version = "3.11"

     

       74
       83
        
       source = { registry = "https://pypi.org/simple" }

     
···

       146
       155
        
       version = "0.1.0"

     

       147
       156
        
       source = { virtual = "." }

     

       148
       157
        
       dependencies = [

     

       158
       158
       +
           { name = "dnspython" },

     

       149
       159
        
           { name = "python-magic" },

     

       150
       160
        
           { name = "requests" },

     

       151
       161
        
           { name = "websockets" },

     
···

       153
       163
        
       

     

       154
       164
        
       [package.metadata]

     

       155
       165
        
       requires-dist = [

     

       166
       166
       +
           { name = "dnspython", specifier = ">=2.8.0" },

     

       156
       167
        
           { name = "python-magic", specifier = ">=0.4.27" },

     

       157
       168
        
           { name = "requests", specifier = ">=2.32.5" },

     

       158
       169
        
           { name = "websockets", specifier = ">=15.0.1" },