commit 9b708c66fbc1457d78e406bc29624e3a389e9e6b · hauleth.dev/esl_hn

+20 -8

lib/esl_hn/cache.ex

···

       10
       10
        
         end

     

       11
       11
        
       

     

       12
       12
        
         def fetch(tid, key) do

     

       13
       13
       -
           case :ets.lookup(tid, key) do

     

       14
       14
       -
             [{^key, data}] -> {:ok, data}

     

       15
       15
       -
             _ -> :error

     

       16
       16
       -
           end

     

       13
       13
       +
           meta = %{key: key, tid: tid}

     

       14
       14
       +
       

     

       15
       15
       +
           :telemetry.span([:esl_hn, :cache, :fetch], meta, fn ->

     

       16
       16
       +
             case :ets.lookup(tid, key) do

     

       17
       17
       +
               [{^key, data}] -> {{:ok, data}, %{hit: 1}, meta}

     

       18
       18
       +
               _ -> {:error, %{miss: 1}, meta}

     

       19
       19
       +
             end

     

       20
       20
       +
           end)

     

       17
       21
        
         end

     

       18
       22
        
       

     

       19
       23
        
         def get(tid, key, default \\ nil) do

     
···

       24
       28
        
         end

     

       25
       29
        
       

     

       26
       30
        
         def write(tid, key, data) do

     

       27
       27
       -
           true = :ets.insert(tid, {key, data})

     

       31
       31
       +
           meta = %{key: key, tid: tid}

     

       28
       32
        
       

     

       29
       29
       -
           :ok

     

       33
       33
       +
           :telemetry.span([:esl_hn, :cache, :write], meta, fn ->

     

       34
       34
       +
             true = :ets.insert(tid, {key, data})

     

       35
       35
       +
       

     

       36
       36
       +
             {:ok, %{count: 1}, meta}

     

       37
       37
       +
           end)

     

       30
       38
        
         end

     

       31
       39
        
       

     

       32
       40
        
         def write_all(tid, data) do

     

       33
       33
       -
           true = :ets.insert(tid, data)

     

       41
       41
       +
           meta = %{tid: tid}

     

       42
       42
       +
       

     

       43
       43
       +
           :telemetry.span([:esl_hn, :cache, :write], meta, fn ->

     

       44
       44
       +
             true = :ets.insert(tid, data)

     

       34
       45
        
       

     

       35
       35
       -
           :ok

     

       46
       46
       +
             {:ok, %{count: length(data)}, meta}

     

       47
       47
       +
           end)

     

       36
       48
        
         end

     

       37
       49
        
       

     

       38
       50
        
         def prune(tid, id) do

+12 -4

lib/esl_hn/hn.ex

···

       11
       11
        
         end

     

       12
       12
        
       

     

       13
       13
        
         def story(id, opts \\ []) do

     

       14
       14
       -
           with {:ok, body} <- get("item/#{id}.json", opts) do

     

       14
       14
       +
           with {:ok, body} <- get("item/:id.json", [id: id], opts) do

     

       15
       15
        
             Story.changeset(body)

     

       16
       16
        
             |> Ecto.Changeset.apply_action(:create)

     

       17
       17
        
           end

     
···

       32
       32
        
       

     

       33
       33
        
         @base_url URI.new!("https://hacker-news.firebaseio.com/v0/")

     

       34
       34
        
       

     

       35
       35
       -
         defp get(path, opts) do

     

       36
       36
       -
           base_url = opts[:base_url] || @base_url

     

       35
       35
       +
         defp get(path, params \\ [], opts) do

     

       36
       36
       +
           opts =

     

       37
       37
       +
             Keyword.merge(

     

       38
       38
       +
               [

     

       39
       39
       +
                 base_url: @base_url,

     

       40
       40
       +
                 path_params: params

     

       41
       41
       +
               ],

     

       42
       42
       +
               opts

     

       43
       43
       +
             )

     

       37
       44
        
       

     

       38
       38
       -
           Req.new(base_url: base_url)

     

       45
       45
       +
           Req.new(opts)

     

       46
       46
       +
           |> ReqTelemetry.attach()

     

       39
       47
        
           |> Req.get(url: path)

     

       40
       48
        
           |> case do

     

       41
       49
        
             {:ok, %Req.Response{status: 200, body: body}} ->

+14 -7

lib/esl_hn/refresher.ex

···

       32
       32
        
       

     

       33
       33
        
         @impl true

     

       34
       34
        
         def handle_continue(:refresh, state) do

     

       35
       35
       +
           ids = fetch(state)

     

       36
       36
       +
       

     

       37
       37
       +
           Process.send_after(self(), :refresh, state.refresh)

     

       38
       38
       +
       

     

       39
       39
       +
           {:noreply, %{state | ids: ids}}

     

       40
       40
       +
         end

     

       41
       41
       +
       

     

       42
       42
       +
         defp fetch(state) do

     

       35
       43
        
           {mod, tid} = state.cache

     

       36
       44
        
       

     

       37
       37
       -
           ids =

     

       45
       45
       +
           :telemetry.span([:esl_hn, :refresh], %{cache: mod}, fn ->

     

       38
       46
        
             with {:ok, ids} <- Hn.top_stories_ids(state.req_opts),

     

       39
       47
        
                  new_set = MapSet.new(ids),

     

       40
       48
        
                  new = MapSet.difference(new_set, state.ids),

     
···

       49
       57
        
       

     

       50
       58
        
               EslHn.broadcast_new(for story <- stories, story.id in new, do: story)

     

       51
       59
        
       

     

       52
       52
       -
               new_set

     

       60
       60
       +
               {new_set, %{new: MapSet.size(new), old: MapSet.size(old)},

     

       61
       61
       +
                %{cache: mod}}

     

       53
       62
        
             else

     

       54
       54
       -
               _ -> state.ids

     

       63
       63
       +
               _ ->

     

       64
       64
       +
                 {state.ids, %{failed: 1}, %{cache: mod}}

     

       55
       65
        
             end

     

       56
       56
       -
       

     

       57
       57
       -
           Process.send_after(self(), :refresh, state.refresh)

     

       58
       58
       -
       

     

       59
       59
       -
           {:noreply, %{state | ids: ids}}

     

       66
       66
       +
           end)

     

       60
       67
        
         end

     

       61
       68
        
       end

+13

lib/esl_hn_web/telemetry.ex

···

       47
       47
        
               tags: [:event],

     

       48
       48
        
               unit: {:native, :millisecond}

     

       49
       49
        
             ),

     

       50
       50
       +
             summary("esl_hn.refresh.stop.duration",

     

       51
       51
       +
               unit: {:native, :millisecond}

     

       52
       52
       +
             ),

     

       53
       53
       +
             sum("esl_hn.refresh.stop.new"),

     

       54
       54
       +
             sum("esl_hn.refresh.stop.old"),

     

       55
       55
       +
             counter("esl_hn.refresh.stop.failed"),

     

       56
       56
       +
             sum("esl_hn.cache.fetch.stop.hit"),

     

       57
       57
       +
             sum("esl_hn.cache.fetch.stop.miss"),

     

       58
       58
       +
             sum("esl_hn.cache.write.stop.count"),

     

       59
       59
       +
             summary("req.request.pipeline.stop.duration",

     

       60
       60
       +
               tags: [:url],

     

       61
       61
       +
               unit: {:native, :millisecond}

     

       62
       62
       +
             ),

     

       50
       63
        
       

     

       51
       64
        
             # VM Metrics

     

       52
       65
        
             summary("vm.memory.total", unit: {:byte, :kilobyte}),

+3 -1

mix.exs

···

       16
       16
        
         def application do

     

       17
       17
        
           [

     

       18
       18
        
             mod: {EslHn.Application, []},

     

       19
       19
       -
             extra_applications: [:logger, :runtime_tools]

     

       19
       19
       +
             extra_applications: [:logger, :runtime_tools, :os_mon]

     

       20
       20
        
           ]

     

       21
       21
        
         end

     

       22
       22
        
       

     
···

       30
       30
        
       

     

       31
       31
        
             # HackerNews client

     

       32
       32
        
             {:req, "~> 0.5.15"},

     

       33
       33
       +
             {:req_telemetry,

     

       34
       34
       +
              github: "hauleth/req_telemetry", ref: "template-paths-as-metadata"},

     

       33
       35
        
             {:ecto, "~> 3.13"},

     

       34
       36
        
       

     

       35
       37
        
             # Monitoring

+1

mix.lock

···

       22
       22
        
         "plug": {:hex, :plug, "1.18.1", "5067f26f7745b7e31bc3368bc1a2b818b9779faa959b49c934c17730efc911cf", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "57a57db70df2b422b564437d2d33cf8d33cd16339c1edb190cd11b1a3a546cc2"},

     

       23
       23
        
         "plug_crypto": {:hex, :plug_crypto, "2.1.1", "19bda8184399cb24afa10be734f84a16ea0a2bc65054e23a62bb10f06bc89491", [:mix], [], "hexpm", "6470bce6ffe41c8bd497612ffde1a7e4af67f36a15eea5f921af71cf3e11247c"},

     

       24
       24
        
         "req": {:hex, :req, "0.5.15", "662020efb6ea60b9f0e0fac9be88cd7558b53fe51155a2d9899de594f9906ba9", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "a6513a35fad65467893ced9785457e91693352c70b58bbc045b47e5eb2ef0c53"},

     

       25
       25
       +
         "req_telemetry": {:git, "https://github.com/hauleth/req_telemetry.git", "449ddbc2f409fb2ea9efe4ad2a2b229cd01aa153", [ref: "template-paths-as-metadata"]},

     

       25
       26
        
         "stream_data": {:hex, :stream_data, "1.2.0", "58dd3f9e88afe27dc38bef26fce0c84a9e7a96772b2925c7b32cd2435697a52b", [:mix], [], "hexpm", "eb5c546ee3466920314643edf68943a5b14b32d1da9fe01698dc92b73f89a9ed"},

     

       26
       27
        
         "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"},

     

       27
       28
        
         "telemetry_metrics": {:hex, :telemetry_metrics, "1.1.0", "5bd5f3b5637e0abea0426b947e3ce5dd304f8b3bc6617039e2b5a008adc02f8f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7b79e8ddfde70adb6db8a6623d1778ec66401f366e9a8f5dd0955c56bc8ce67"},

+15

test/esl_hn/hn_test.exs

···

       35
       35
        
         end

     

       36
       36
        
       

     

       37
       37
        
         describe "story/2" do

     

       38
       38
       +
           test "returns error on non 200 response" do

     

       39
       39
       +
             data = %{

     

       40
       40
       +
               id: 2137,

     

       41
       41
       +
               title: "Foo"

     

       42
       42
       +
             }

     

       43
       43
       +
       

     

       44
       44
       +
             TestServer.add("/item/2137.json",

     

       45
       45
       +
               to: fn conn ->

     

       46
       46
       +
                 Phoenix.Controller.json(conn, data)

     

       47
       47
       +
               end

     

       48
       48
       +
             )

     

       49
       49
       +
       

     

       50
       50
       +
             assert {:ok, %{title: "Foo"}} =

     

       51
       51
       +
                      @subject.story(2137, base_url: TestServer.url())

     

       52
       52
       +
           end

     

       38
       53
        
         end

     

       39
       54
        
       end