commit b1a96bbbf22c2e375080773a366f6e9dbf7cd3ba · pyrox.dev/nixpkgs

+22

doc/default.nix

···

       149
       149
        
           echo "doc manual $dest ${common.indexPath}" >> $out/nix-support/hydra-build-products

     

       150
       150
        
           echo "doc manual $dest nixpkgs-manual.epub" >> $out/nix-support/hydra-build-products

     

       151
       151
        
         '';

     

       152
       152
       +
       

     

       153
       153
       +
         passthru.tests.manpage-urls = with pkgs; testers.invalidateFetcherByDrvHash

     

       154
       154
       +
           ({ name ? "manual_check-manpage-urls"

     

       155
       155
       +
            , script

     

       156
       156
       +
            , urlsFile

     

       157
       157
       +
            }: runCommand name {

     

       158
       158
       +
             nativeBuildInputs = [

     

       159
       159
       +
               cacert

     

       160
       160
       +
               (python3.withPackages (p: with p; [

     

       161
       161
       +
                 aiohttp

     

       162
       162
       +
                 rich

     

       163
       163
       +
                 structlog

     

       164
       164
       +
               ]))

     

       165
       165
       +
             ];

     

       166
       166
       +
             outputHash = "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=";  # Empty output

     

       167
       167
       +
           } ''

     

       168
       168
       +
             python3 ${script} ${urlsFile}

     

       169
       169
       +
             touch $out

     

       170
       170
       +
           '') {

     

       171
       171
       +
             script = ./tests/manpage-urls.py;

     

       172
       172
       +
             urlsFile = ./manpage-urls.json;

     

       173
       173
       +
           };

     

       152
       174
        
       }

+107

doc/tests/manpage-urls.py

···

       1
       1
       +
       #! /usr/bin/env nix-shell

     

       2
       2
       +
       #! nix-shell -i "python3 -I" -p "python3.withPackages(p: with p; [ aiohttp rich structlog ])"

     

       3
       3
       +
       

     

       4
       4
       +
       from argparse import ArgumentParser

     

       5
       5
       +
       from collections import defaultdict

     

       6
       6
       +
       from enum import IntEnum

     

       7
       7
       +
       from http import HTTPStatus

     

       8
       8
       +
       from pathlib import Path

     

       9
       9
       +
       import asyncio, json, logging

     

       10
       10
       +
       

     

       11
       11
       +
       import aiohttp, structlog

     

       12
       12
       +
       from structlog.contextvars import bound_contextvars as log_context

     

       13
       13
       +
       

     

       14
       14
       +
       

     

       15
       15
       +
       LogLevel = IntEnum('LogLevel', {

     

       16
       16
       +
           lvl: getattr(logging, lvl)

     

       17
       17
       +
           for lvl in ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL')

     

       18
       18
       +
       })

     

       19
       19
       +
       LogLevel.__str__ = lambda self: self.name

     

       20
       20
       +
       

     

       21
       21
       +
       

     

       22
       22
       +
       EXPECTED_STATUS=frozenset((

     

       23
       23
       +
           HTTPStatus.OK, HTTPStatus.FOUND,

     

       24
       24
       +
           HTTPStatus.NOT_FOUND,

     

       25
       25
       +
       ))

     

       26
       26
       +
       

     

       27
       27
       +
       async def check(session, manpage: str, url: str) -> HTTPStatus:

     

       28
       28
       +
           with log_context(manpage=manpage, url=url):

     

       29
       29
       +
               logger.debug("Checking")

     

       30
       30
       +
               async with session.head(url) as resp:

     

       31
       31
       +
                   st = HTTPStatus(resp.status)

     

       32
       32
       +
                   match st:

     

       33
       33
       +
                       case HTTPStatus.OK | HTTPStatus.FOUND:

     

       34
       34
       +
                           logger.debug("OK!")

     

       35
       35
       +
                       case HTTPStatus.NOT_FOUND:

     

       36
       36
       +
                           logger.error("Broken link!")

     

       37
       37
       +
                       case _ if st < 400:

     

       38
       38
       +
                           logger.info("Unexpected code", status=st)

     

       39
       39
       +
                       case _ if 400 <= st < 600:

     

       40
       40
       +
                           logger.warn("Unexpected error", status=st)

     

       41
       41
       +
       

     

       42
       42
       +
                   return st

     

       43
       43
       +
       

     

       44
       44
       +
       async def main(urls_path):

     

       45
       45
       +
           logger.info(f"Parsing {urls_path}")

     

       46
       46
       +
           with urls_path.open() as urls_file:

     

       47
       47
       +
               urls = json.load(urls_file)

     

       48
       48
       +
       

     

       49
       49
       +
           count = defaultdict(lambda: 0)

     

       50
       50
       +
       

     

       51
       51
       +
           logger.info(f"Checking URLs from {urls_path}")

     

       52
       52
       +
           async with aiohttp.ClientSession() as session:

     

       53
       53
       +
               for status in asyncio.as_completed([

     

       54
       54
       +
                   check(session, manpage, url)

     

       55
       55
       +
                   for manpage, url in urls.items()

     

       56
       56
       +
               ]):

     

       57
       57
       +
                   count[await status]+=1

     

       58
       58
       +
       

     

       59
       59
       +
           ok = count[HTTPStatus.OK] + count[HTTPStatus.FOUND]

     

       60
       60
       +
           broken = count[HTTPStatus.NOT_FOUND]

     

       61
       61
       +
           unknown = sum(c for st, c in count.items() if st not in EXPECTED_STATUS)

     

       62
       62
       +
           logger.info(f"Done: {broken} broken links, "

     

       63
       63
       +
                       f"{ok} correct links, and {unknown} unexpected status")

     

       64
       64
       +
       

     

       65
       65
       +
           return count

     

       66
       66
       +
       

     

       67
       67
       +
       

     

       68
       68
       +
       def parse_args(args=None):

     

       69
       69
       +
           parser = ArgumentParser(

     

       70
       70
       +
               prog = 'check-manpage-urls',

     

       71
       71
       +
               description = 'Check the validity of the manpage URLs linked in the nixpkgs manual',

     

       72
       72
       +
           )

     

       73
       73
       +
           parser.add_argument(

     

       74
       74
       +
               '-l', '--log-level',

     

       75
       75
       +
               default = os.getenv('LOG_LEVEL', 'INFO'),

     

       76
       76
       +
               type = lambda s: LogLevel[s],

     

       77
       77
       +
               choices = list(LogLevel),

     

       78
       78
       +
           )

     

       79
       79
       +
           parser.add_argument(

     

       80
       80
       +
               'file',

     

       81
       81
       +
               type = Path,

     

       82
       82
       +
               nargs = '?',

     

       83
       83
       +
           )

     

       84
       84
       +
       

     

       85
       85
       +
           return parser.parse_args(args)

     

       86
       86
       +
       

     

       87
       87
       +
       

     

       88
       88
       +
       if __name__ == "__main__":

     

       89
       89
       +
           import os, sys

     

       90
       90
       +
       

     

       91
       91
       +
           args = parse_args()

     

       92
       92
       +
       

     

       93
       93
       +
           structlog.configure(

     

       94
       94
       +
               wrapper_class=structlog.make_filtering_bound_logger(args.log_level),

     

       95
       95
       +
           )

     

       96
       96
       +
           logger = structlog.getLogger("check-manpage-urls.py")

     

       97
       97
       +
       

     

       98
       98
       +
           urls_path = args.file

     

       99
       99
       +
           if urls_path is None:

     

       100
       100
       +
               REPO_ROOT = Path(__file__).parent.parent.parent.parent

     

       101
       101
       +
               logger.info(f"Assuming we are in a nixpkgs repo rooted at {REPO_ROOT}")

     

       102
       102
       +
       

     

       103
       103
       +
               urls_path = REPO_ROOT / 'doc' / 'manpage-urls.json'

     

       104
       104
       +
       

     

       105
       105
       +
           count = asyncio.run(main(urls_path))

     

       106
       106
       +
       

     

       107
       107
       +
           sys.exit(0 if count[HTTPStatus.NOT_FOUND] == 0 else 1)