commit 7e647c4b0ef5705c11eb9cf894a811539f831349 · zenfyr.dev/xpost

pyproject.toml

···

       6
        
       requires-python = ">=3.12"

     

       7
        
       dependencies = [

     

       8
        
           "dnspython>=2.8.0",

     

       0
        
       
     

       9
        
           "python-magic>=0.4.27",

     

       10
        
           "requests>=2.32.5",

     

       11
        
           "websockets>=15.0.1",

···

       6
        
       requires-python = ">=3.12"

     

       7
        
       dependencies = [

     

       8
        
           "dnspython>=2.8.0",

     

       9
       +
           "grapheme>=0.6.0",

     

       10
        
           "python-magic>=0.4.27",

     

       11
        
           "requests>=2.32.5",

     

       12
        
           "websockets>=15.0.1",

+33 -48

util/markdown.py

···

       0
        
       
     

       1
        
       import re

     

       2
        
       import cross.fragments as f

     

       3
        
       from util.html import HTMLToFragmentsParser

     
···

       30
        
               total: int = len(markdown)

     

       31
        
       

     

       32
        
               # no match == processed fragments

     

       33
       -
               events: list[tuple[int, int, re.Match[str] | None, str]] = []

     

       34
       -
               events.extend([(fg.start, fg.end, None, "html") for fg in fragments])

     

       35
        
               while index < total:

     

       36
        
                   ch = markdown[index]

     

       37
        
                   rmatch = None

     
···

       77
        
                       )

     

       78
        
                   last_end = end

     

       79
        
       

     

       80
       -
               def update_fragments(start: int, s, offset: int):

     

       81
       -
                   nonlocal fragments

     

       82
        
       

     

       83
       -
                   for fg in fragments:

     

       84
       -
                       if fg != s and fg.start >= start:

     

       85
       -
                           fg.start += offset

     

       86
       -
                           fg.end += offset

     

       87
        
       

     

       88
       -
               new_text = ""

     

       89
       -
               last_pos = 0

     

       90
        
               for start, end, rmatch, event in events:

     

       91
       -
                   if start > last_pos:

     

       92
       -
                       new_text += markdown[last_pos:start]

     

       93
        
       

     

       94
       -
                   if not rmatch:

     

       95
       -
                       new_text += markdown[start:end]

     

       96
       -
                       last_pos = end

     

       0
        
       
     

       0
        
       
     

       97
        
                       continue

     

       98
        
       

     

       0
        
       
     

       0
        
       
     

       99
        
                   match event:

     

       100
        
                       case "inline_link":

     

       101
        
                           label = rmatch.group(1)

     

       102
        
                           href = rmatch.group(2)

     

       103
       -
                           fg = f.LinkFragment(start=start, end=start + len(label), url=href)

     

       104
       -
                           fragments.append(fg)

     

       105
       -
                           update_fragments(start, fg, -(end - (start + len(label))))

     

       106
       -
                           new_text += label

     

       107
       -
                       # case "autolink":

     

       108
       -
                       #    url = rmatch.group(0)

     

       109
       -
                       #    fg = f.LinkFragment(start=start, end=end - 2, url=url)

     

       110
       -
                       #    fragments.append(fg)

     

       111
       -
                       #    update_fragments(start, fg, -2)

     

       112
       -
                       #    new_text += url

     

       113
        
                       case "hashtag":

     

       114
       -
                           tag = rmatch.group(0)

     

       115
       -
                           fragments.append(

     

       116
       -
                               f.TagFragment(

     

       117
       -
                                   start=start,

     

       118
       -
                                   end=end,

     

       119
       -
                                   tag=tag[1:] if tag.startswith("#") else tag,

     

       120
       -
                               )

     

       121
       -
                           )

     

       122
       -
                           new_text += markdown[start:end]

     

       123
        
                       case "mention":

     

       124
        
                           mention = rmatch.group(0)

     

       125
       -
                           fragments.append(

     

       126
       -
                               f.MentionFragment(

     

       127
       -
                                   start=start,

     

       128
       -
                                   end=end,

     

       129
       -
                                   uri=mention[1:] if mention.startswith("@") else mention,

     

       130
       -
                               )

     

       131
       -
                           )

     

       132
       -
                           new_text += markdown[start:end]

     

       133
        
                       case "url":

     

       134
        
                           url = rmatch.group(0)

     

       135
       -
                           fragments.append(f.LinkFragment(start=start, end=end, url=url))

     

       136
       -
                           new_text += markdown[start:end]

     

       137
        
                       case _:

     

       138
        
                           pass

     

       139
       -
                   last_pos = end

     

       140
       -
               if last_pos < len(markdown):

     

       141
       -
                   new_text += markdown[last_pos:]

     

       142
        
       

     

       143
       -
               return new_text, fragments

···

       1
       +
       from dataclasses import replace

     

       2
        
       import re

     

       3
        
       import cross.fragments as f

     

       4
        
       from util.html import HTMLToFragmentsParser

     
···

       31
        
               total: int = len(markdown)

     

       32
        
       

     

       33
        
               # no match == processed fragments

     

       34
       +
               events: list[tuple[int, int, re.Match[str] | f.Fragment, str]] = []

     

       35
       +
               events.extend([(fg.start, fg.end, fg, "html") for fg in fragments])

     

       36
        
               while index < total:

     

       37
        
                   ch = markdown[index]

     

       38
        
                   rmatch = None

     
···

       78
        
                       )

     

       79
        
                   last_end = end

     

       80
        
       

     

       81
       +
               ntext: list[str] = []

     

       82
       +
               nfragments: list[f.Fragment] = []

     

       83
        
       

     

       84
       +
               offset: int = 0

     

       85
       +
               last_index: int = 0

     

       0
        
       
     

       0
        
       
     

       86
        
       

     

       87
       +
               events.sort(key=lambda x: x[0])

     

       0
        
       
     

       88
        
               for start, end, rmatch, event in events:

     

       89
       +
                   ntext.append(markdown[last_index:start])

     

       0
        
       
     

       90
        
       

     

       91
       +
                   if isinstance(rmatch, f.Fragment):

     

       92
       +
                       ntext.append(markdown[start:end])

     

       93
       +
                       nfg = replace(rmatch, start=start + offset, end=end + offset)

     

       94
       +
                       nfragments.append(nfg)

     

       95
       +
                       last_index = end

     

       96
        
                       continue

     

       97
        
       

     

       98
       +
                   nstart = start + offset

     

       99
       +
                   nend = end + offset

     

       100
        
                   match event:

     

       101
        
                       case "inline_link":

     

       102
        
                           label = rmatch.group(1)

     

       103
        
                           href = rmatch.group(2)

     

       104
       +
                           ntext.append(label)

     

       105
       +
       

     

       106
       +
                           delta = len(label) - (end - start)

     

       107
       +
                           offset += delta

     

       108
       +
       

     

       109
       +
                           nfragments.append(f.LinkFragment(start=nstart, end=nstart + len(label), url=href))

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       110
        
                       case "hashtag":

     

       111
       +
                           tag = rmatch.group(1)

     

       112
       +
                           ntext.append(markdown[start:end])

     

       113
       +
                           nfragments.append(f.TagFragment(start=nstart, end=nend, tag=tag))

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       114
        
                       case "mention":

     

       115
        
                           mention = rmatch.group(0)

     

       116
       +
                           ntext.append(markdown[start:end])

     

       117
       +
                           mention = mention[1:] if mention.startswith("@") else mention

     

       118
       +
                           nfragments.append(f.MentionFragment(start=nstart, end=nend, uri=mention))

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       119
        
                       case "url":

     

       120
        
                           url = rmatch.group(0)

     

       121
       +
                           ntext.append(markdown[start:end])

     

       122
       +
                           nfragments.append(f.LinkFragment(start=nstart, end=nend, url=url))

     

       123
        
                       case _:

     

       124
        
                           pass

     

       125
       +
                   last_index = end

     

       126
       +
               ntext.append(markdown[last_index:])

     

       0
        
       
     

       127
        
       

     

       128
       +
               return ''.join(ntext), nfragments

+72

util/splitter.py

···

       1
       +
       import grapheme

     

       2
       +
       from cross.fragments import Fragment, LinkFragment

     

       3
       +
       from dataclasses import replace

     

       4
       +
       

     

       5
       +
       

     

       6
       +
       def canonical_label(label: str | None, href: str):

     

       7
       +
           if not label or label == href:

     

       8
       +
               return True

     

       9
       +
       

     

       10
       +
           split = href.split("://", 1)

     

       11
       +
           if len(split) > 1:

     

       12
       +
               if split[1] == label:

     

       13
       +
                   return True

     

       14
       +
       

     

       15
       +
           return False

     

       16
       +
       

     

       17
       +
       

     

       18
       +
       class FragmentSplitter:

     

       19
       +
           def __init__(self, climit: int, urllen: int):

     

       20
       +
               self.climit: int = climit

     

       21
       +
               self.urllen: int = urllen

     

       22
       +
       

     

       23
       +
           def normalize_link(self, label: str, url: str) -> str:

     

       24
       +
               #if canonical_label(label, url):

     

       25
       +
               #    if self.urltrunc == "dotted":

     

       26
       +
                   #        nlabel = url.split("://", 1)[1]

     

       27
       +
                   #        if len(nlabel) <= self.urllen:

     

       28
       +
                       #            return nlabel

     

       29
       +
                       #        return nlabel[: self.urllen - 1] + "…"

     

       30
       +
               return label

     

       31
       +
       

     

       32
       +
           def url_normalize(

     

       33
       +
               self, text: str, fragments: list[Fragment]

     

       34
       +
           ) -> tuple[str, list[Fragment]]:

     

       35
       +
               if self.urllen == -1:

     

       36
       +
                   return text, fragments

     

       37
       +
       

     

       38
       +
               ntext: list[str] = []

     

       39
       +
               nfragments: list[Fragment] = []

     

       40
       +
       

     

       41
       +
               offset: int = 0

     

       42
       +
               last_index: int = 0

     

       43
       +
       

     

       44
       +
               fragments = [fg for fg in fragments]

     

       45
       +
               fragments.sort(key=lambda x: x.start)

     

       46
       +
       

     

       47
       +
               for fg in fragments:

     

       48
       +
                   ntext.append(text[last_index:fg.start])

     

       49
       +
                   label = text[fg.start:fg.end]

     

       50
       +
                   nlabel = label

     

       51
       +
                   if isinstance(fg, LinkFragment):

     

       52
       +
                       nlabel = self.normalize_link(nlabel, fg.url)

     

       53
       +
                   ntext.append(nlabel)

     

       54
       +
       

     

       55
       +
                   nfg = replace(fg, start=fg.start + offset)

     

       56
       +
                   change = len(nlabel) - len(label)

     

       57
       +
                   offset += change

     

       58
       +
                   nfg = replace(nfg, end=fg.end + offset)

     

       59
       +
       

     

       60
       +
                   nfragments.append(nfg)

     

       61
       +
                   last_index = fg.end

     

       62
       +
       

     

       63
       +
               ntext.append(text[last_index:])

     

       64
       +
       

     

       65
       +
               return ''.join(ntext), nfragments

     

       66
       +
       

     

       67
       +
           def split(

     

       68
       +
               self, text: str, fragments: list[Fragment]

     

       69
       +
           ) -> list[tuple[str, list[Fragment]]]:

     

       70
       +
               text, fragments = self.url_normalize(text, fragments)

     

       71
       +
               if grapheme.length(text) <= self.climit:

     

       72
       +
                   return [(text, fragments)]

uv.lock

···

       78
        
       ]

     

       79
        
       

     

       80
        
       [[package]]

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       81
        
       name = "idna"

     

       82
        
       version = "3.11"

     

       83
        
       source = { registry = "https://pypi.org/simple" }

     
···

       156
        
       source = { virtual = "." }

     

       157
        
       dependencies = [

     

       158
        
           { name = "dnspython" },

     

       0
        
       
     

       159
        
           { name = "python-magic" },

     

       160
        
           { name = "requests" },

     

       161
        
           { name = "websockets" },

     
···

       164
        
       [package.metadata]

     

       165
        
       requires-dist = [

     

       166
        
           { name = "dnspython", specifier = ">=2.8.0" },

     

       0
        
       
     

       167
        
           { name = "python-magic", specifier = ">=0.4.27" },

     

       168
        
           { name = "requests", specifier = ">=2.32.5" },

     

       169
        
           { name = "websockets", specifier = ">=15.0.1" },

···

       78
        
       ]

     

       79
        
       

     

       80
        
       [[package]]

     

       81
       +
       name = "grapheme"

     

       82
       +
       version = "0.6.0"

     

       83
       +
       source = { registry = "https://pypi.org/simple" }

     

       84
       +
       sdist = { url = "https://files.pythonhosted.org/packages/ce/e7/bbaab0d2a33e07c8278910c1d0d8d4f3781293dfbc70b5c38197159046bf/grapheme-0.6.0.tar.gz", hash = "sha256:44c2b9f21bbe77cfb05835fec230bd435954275267fea1858013b102f8603cca", size = 207306, upload-time = "2020-03-07T17:13:55.492Z" }

     

       85
       +
       

     

       86
       +
       [[package]]

     

       87
        
       name = "idna"

     

       88
        
       version = "3.11"

     

       89
        
       source = { registry = "https://pypi.org/simple" }

     
···

       162
        
       source = { virtual = "." }

     

       163
        
       dependencies = [

     

       164
        
           { name = "dnspython" },

     

       165
       +
           { name = "grapheme" },

     

       166
        
           { name = "python-magic" },

     

       167
        
           { name = "requests" },

     

       168
        
           { name = "websockets" },

     
···

       171
        
       [package.metadata]

     

       172
        
       requires-dist = [

     

       173
        
           { name = "dnspython", specifier = ">=2.8.0" },

     

       174
       +
           { name = "grapheme", specifier = ">=0.6.0" },

     

       175
        
           { name = "python-magic", specifier = ">=0.4.27" },

     

       176
        
           { name = "requests", specifier = ">=2.32.5" },

     

       177
        
           { name = "websockets", specifier = ">=15.0.1" },