My 1billion row challenge solutions in various languages
treybastian.com
1import 'dart:collection';
2import 'dart:convert';
3import 'dart:io';
4import 'dart:isolate';
5
6final newLine = 10;
7final semiColon = 59;
8final minus = 45;
9
10class Result {
11 double total = 0;
12 double? min = null;
13 double? max = null;
14 double count = 0;
15
16 void incrementCount() {
17 this.count += 1;
18 }
19
20 void addToTotal(double value) {
21 this.total += value;
22 }
23
24 void processMin(double value) {
25 if (this.min == null || this.min! > value) {
26 this.min = value;
27 }
28 }
29
30 void processMax(double value) {
31 if (this.max == null || this.max! < value) {
32 this.max = value;
33 }
34 }
35
36 double avg() {
37 return (this.total / this.count);
38 }
39}
40
41final Map<String, Result> res =
42 SplayTreeMap<String, Result>((a, b) => a.compareTo(b));
43
44void main() async {
45 final file = File('measurements.txt').openSync();
46 final fileSize = file.lengthSync();
47 final maxLineLength = 106;
48
49 final threadCount = Platform.numberOfProcessors;
50
51 final chunkSize = fileSize ~/ threadCount;
52 final offsets = <int>[];
53
54 int offset = 0;
55 for (int i = 0; i < threadCount; i++) {
56 offsets.add(offset);
57
58 offset += chunkSize;
59
60 if (offset >= fileSize) {
61 break;
62 }
63 file.setPositionSync(offset);
64 var line = file.readSync(maxLineLength);
65
66 offset += line.indexOf(newLine) + 1;
67 }
68 final processor =
69 List<Future<Map<String, Result>>>.generate(offsets.length, (int i) {
70 var end = offsets.length == i + 1 ? fileSize : offsets[i + 1];
71 return Isolate.run(() => processChunk(offsets[i], end));
72 });
73 for (var res in await Future.wait(processor)) {
74 addResults(res);
75 }
76
77 printResults();
78}
79
80void addResults(Map<String, Result> results) {
81 results.forEach((key, val) {
82 res.putIfAbsent(key, () => new Result())
83 ..addToTotal(val.total)
84 ..processMin(val.min!)
85 ..processMax(val.max!)
86 ..count += val.count;
87 });
88}
89
90Future<Map<String, Result>> processChunk(int start, int end) async {
91 final res = new HashMap<String, Result>();
92 final file = File("measurements.txt");
93 List<int> buffer = List<int>.filled(106, 0);
94 String line = '';
95 int ct = 0;
96 List<String> split;
97 await for (var chunk in file.openRead(start, end)) {
98 for (var b in chunk) {
99 if (b != newLine) {
100 buffer[ct] = b;
101 ct++;
102 } else {
103 line = utf8.decode(buffer.take(ct).toList());
104 split = line.split(';');
105 var value = double.parse(split[1]);
106 res.putIfAbsent(split[0], () => new Result())
107 ..addToTotal(value!)
108 ..processMin(value)
109 ..processMax(value)
110 ..incrementCount();
111 ct = 0;
112 }
113 }
114 }
115 return res;
116}
117
118void printResults() {
119 res.forEach((key, value) {
120 print('$key;${value.min};${value.avg().toStringAsFixed(1)};${value.max}');
121 });
122}