commit a503362c3678f2490073d303750c4f14f1945cf4 · treybastian.com/1brc

+16 -8

src/main/java/dev/morling/onebrc/CalculateAverage_ebarlas.java

···

       51
       51
        
                       try {

     

       52
       52
        
                           var buffer = channel.map(FileChannel.MapMode.READ_ONLY, pStart, pSize);

     

       53
       53
        
                           partitions[pIdx] = processBuffer(buffer, pIdx == 0);

     

       54
       54
       -
                       } catch (IOException e) {

     

       54
       54
       +
                       }

     

       55
       55
       +
                       catch (IOException e) {

     

       55
       56
        
                           throw new RuntimeException(e);

     

       56
       57
        
                       }

     

       57
       58
        
                   };

     
···

       94
       95
        
                           var t = target[j];

     

       95
       96
        
                           if (t == null) {

     

       96
       97
        
                               target[j] = current[j]; // copy ref from current to target

     

       97
       97
       -
                           } else {

     

       98
       98
       +
                           }

     

       99
       99
       +
                           else {

     

       98
       100
        
                               t.min = Math.min(t.min, current[j].min);

     

       99
       101
        
                               t.max = Math.max(t.max, current[j].max);

     

       100
       102
        
                               t.sum += current[j].sum;

     
···

       154
       156
        
                               st = stats[idx] = new Stats(key);

     

       155
       157
        
                           }

     

       156
       158
        
                           readingKey = false;

     

       157
       157
       -
                       } else {

     

       159
       159
       +
                       }

     

       160
       160
       +
                       else {

     

       158
       161
        
                           keyHash = HASH_FACTOR * keyHash + b;

     

       159
       162
        
                       }

     

       160
       160
       -
                   } else {

     

       163
       163
       +
                   }

     

       164
       164
       +
                   else {

     

       161
       165
        
                       if (b == '\n') {

     

       162
       166
        
                           var v = negative ? -val : val;

     

       163
       167
        
                           st.min = Math.min(st.min, v);

     
···

       169
       173
        
                           val = 0;

     

       170
       174
        
                           negative = false;

     

       171
       175
        
                           keyStart = buffer.position();

     

       172
       172
       -
                       } else if (b == '-') {

     

       176
       176
       +
                       }

     

       177
       177
       +
                       else if (b == '-') {

     

       173
       178
        
                           negative = true;

     

       174
       174
       -
                       } else if (b != '.') { // skip '.' since fractional tenth unit after decimal point is assumed

     

       179
       179
       +
                       }

     

       180
       180
       +
                       else if (b != '.') { // skip '.' since fractional tenth unit after decimal point is assumed

     

       175
       181
        
                           val = val * 10 + (b - '0');

     

       176
       182
        
                       }

     

       177
       183
        
                   }

     
···

       187
       193
        
           }

     

       188
       194
        
       

     

       189
       195
        
           private static byte[] readHeader(ByteBuffer buffer) { // read up to and including first newline (or end-of-input)

     

       190
       190
       -
               while (buffer.hasRemaining() && buffer.get() != '\n') ;

     

       196
       196
       +
               while (buffer.hasRemaining() && buffer.get() != '\n')

     

       197
       197
       +
                   ;

     

       191
       198
        
               var header = new byte[buffer.position()];

     

       192
       199
        
               buffer.get(0, header, 0, header.length);

     

       193
       200
        
               return header;

     

       194
       201
        
           }

     

       195
       202
        
       

     

       196
       196
       -
           record Partition(byte[] header, byte[] footer, Stats[] stats) {}

     

       203
       203
       +
           record Partition(byte[] header, byte[] footer, Stats[] stats) {

     

       204
       204
       +
           }

     

       197
       205
        
       

     

       198
       206
        
           private static class Stats { // min, max, and sum values are modeled with integral types that represent tenths of a unit

     

       199
       207
        
               final byte[] key;

+1 -2

src/main/java/dev/morling/onebrc/CalculateAverage_seijikun.java

···

       26
       26
        
       

     

       27
       27
        
           private static final String FILE = "./measurements.txt";

     

       28
       28
        
       

     

       29
       29
       -
       

     

       30
       29
        
           private static class MeasurementAggregator {

     

       31
       30
        
               private double min = Double.POSITIVE_INFINITY;

     

       32
       31
        
               private double max = Double.NEGATIVE_INFINITY;

     
···

       179
       178
        
                   chunkStartPtr = chunkEndPtr;

     

       180
       179
        
               }

     

       181
       180
        
       

     

       182
       182
       -
               try(var executor = Executors.newFixedThreadPool(jobCnt)) {

     

       181
       181
       +
               try (var executor = Executors.newFixedThreadPool(jobCnt)) {

     

       183
       182
        
                   for (int i = 0; i < jobCnt; ++i) {

     

       184
       183
        
                       executor.submit(chunks[i]);

     

       185
       184
        
                   }

+13 -13

src/main/java/dev/morling/onebrc/CalculateAverage_truelive.java

···

       105
       105
        
           }

     

       106
       106
        
       

     

       107
       107
        
           private static Map<String, Measurement> combineMaps(

     

       108
       108
       -
                   final Map<String, Measurement> map1,

     

       109
       109
       -
                   final Map<String, Measurement> map2

     

       110
       110
       -
           ) {

     

       108
       108
       +
                                                               final Map<String, Measurement> map1,

     

       109
       109
       +
                                                               final Map<String, Measurement> map2) {

     

       111
       110
        
               for (final var entry : map2.entrySet()) {

     

       112
       111
        
                   map1.merge(entry.getKey(), entry.getValue(), Measurement::combineWith);

     

       113
       112
        
               }

     
···

       116
       115
        
           }

     

       117
       116
        
       

     

       118
       117
        
           public static void main(final String[] args) throws IOException {

     

       119
       119
       -
               //long before = System.currentTimeMillis();

     

       118
       118
       +
               // long before = System.currentTimeMillis();

     

       120
       119
        
               /**

     

       121
       120
        
                * Shoutout to bjhara

     

       122
       121
        
                */

     
···

       136
       135
        
                           final MappedByteBuffer mbb = in.map(

     

       137
       136
        
                                   FileChannel.MapMode.READ_ONLY,

     

       138
       137
        
                                   start,

     

       139
       139
       -
                                   Math.min(CHUNK_SIZE, total - start)

     

       140
       140
       -
                           );

     

       138
       138
       +
                                   Math.min(CHUNK_SIZE, total - start));

     

       141
       139
        
                           int realEnd = mbb.limit() - 1;

     

       142
       140
        
                           while (mbb.get(realEnd) != '\n') {

     

       143
       141
        
                               realEnd--;

     
···

       148
       146
        
                           start += realEnd;

     

       149
       147
        
       

     

       150
       148
        
                           return mbb;

     

       151
       151
       -
                       } catch (final IOException e) {

     

       149
       149
       +
                       }

     

       150
       150
       +
                       catch (final IOException e) {

     

       152
       151
        
                           throw new RuntimeException(e);

     

       153
       152
        
                       }

     

       154
       153
        
                   }

     

       155
       154
        
               };

     

       156
       155
        
               final Map<String, Measurement> reduce = StreamSupport.stream(Spliterators.spliteratorUnknownSize(

     

       157
       157
       -
                                                                      iterator, Spliterator.IMMUTABLE), true)

     

       158
       158
       -
                                                                    .parallel()

     

       159
       159
       -
                                                                    .map(CalculateAverage_truelive::parseBuffer)

     

       160
       160
       -
                                                                    .reduce(CalculateAverage_truelive::combineMaps).get();

     

       156
       156
       +
                       iterator, Spliterator.IMMUTABLE), true)

     

       157
       157
       +
                       .parallel()

     

       158
       158
       +
                       .map(CalculateAverage_truelive::parseBuffer)

     

       159
       159
       +
                       .reduce(CalculateAverage_truelive::combineMaps).get();

     

       161
       160
        
       

     

       162
       161
        
               System.out.print("{");

     

       163
       162
        
               System.out.print(

     
···

       169
       168
        
                               .collect(Collectors.joining(", ")));

     

       170
       169
        
               System.out.println("}");

     

       171
       170
        
       

     

       172
       172
       -
               //System.out.println("Took: " + (System.currentTimeMillis() - before));

     

       171
       171
       +
               // System.out.println("Took: " + (System.currentTimeMillis() - before));

     

       173
       172
        
       

     

       174
       173
        
           }

     

       175
       174
        
       

     
···

       189
       188
        
                       name = new String(arr, 0, len);

     

       190
       189
        
                       bug.position(pos);

     

       191
       190
        
                       bug.mark();

     

       192
       192
       -
                   } else if (c == '\n') {

     

       191
       191
       +
                   }

     

       192
       192
       +
                   else if (c == '\n') {

     

       193
       193
        
                       final int pos = bug.position();

     

       194
       194
        
                       bug.reset();

     

       195
       195
        
                       final int len = pos - bug.position();

+36 -36

src/main/java/dev/morling/onebrc/CreateMeasurements2.java

···

       35
       35
        
           private static final String FILE = "./measurements2.txt";

     

       36
       36
        
       

     

       37
       37
        
           static class WeatherStation {

     

       38
       38
       -
               final static char[] NUMBERS = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9' };

     

       38
       38
       +
               final static char[] NUMBERS = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' };

     

       39
       39
        
       

     

       40
       40
        
               final String id;

     

       41
       41
        
               final int meanTemperature;

     
···

       58
       58
        
                * @param buffer the buffer to append to

     

       59
       59
        
                */

     

       60
       60
        
               void measurement(final CheaperCharBuffer buffer) {

     

       61
       61
       -
               	// fake -10.9 to +10.9 variance without double operations and rounding

     

       62
       62
       -
               	// gives us -10 to +10

     

       61
       61
       +
                   // fake -10.9 to +10.9 variance without double operations and rounding

     

       62
       62
       +
                   // gives us -10 to +10

     

       63
       63
        
                   int m = meanTemperature + (r.nextInt(21) - 10);

     

       64
       64
        
                   // gives us a decimal digit 0 to 9 as char

     

       65
       65
        
                   char d = NUMBERS[r.nextInt(10)];

     
···

       569
       569
        
       

     

       570
       570
        
               // manual loop unroll for less jumps

     

       571
       571
        
               for (int i = 0; i < count; i = i + 8) {

     

       572
       572
       -
               	{

     

       573
       573
       -
               		// try to fill teh cpu pipeline as much as possible with

     

       574
       574
       -
               		// independent operations

     

       575
       575
       -
               		int s1 = r1.nextInt(stationCount);

     

       576
       576
       -
               		int s2 = r2.nextInt(stationCount);

     

       577
       577
       -
               		int s3 = r3.nextInt(stationCount);

     

       578
       578
       -
               		int s4 = r4.nextInt(stationCount);

     

       579
       579
       -
               		// get us the ojects one after the other to have the array

     

       580
       580
       -
               		// in our L1 cache and not push it out with other data

     

       581
       581
       -
               		var w1 = stations.get(s1);

     

       582
       582
       -
               		var w2 = stations.get(s2);

     

       583
       583
       -
               		var w3 = stations.get(s3);

     

       584
       584
       -
               		var w4 = stations.get(s4);

     

       585
       585
       -
               		// write our data to our buffer

     

       586
       586
       -
               		w1.measurement(sb);

     

       587
       587
       -
               		w2.measurement(sb);

     

       588
       588
       -
               		w3.measurement(sb);

     

       589
       589
       -
               		w4.measurement(sb);

     

       590
       590
       -
               	}

     

       591
       591
       -
               	{

     

       592
       592
       -
               		int s1 = r1.nextInt(stationCount);

     

       593
       593
       -
               		int s2 = r2.nextInt(stationCount);

     

       594
       594
       -
               		int s3 = r3.nextInt(stationCount);

     

       595
       595
       -
               		int s4 = r4.nextInt(stationCount);

     

       596
       596
       -
               		var w1 = stations.get(s1);

     

       597
       597
       -
               		var w2 = stations.get(s2);

     

       598
       598
       -
               		var w3 = stations.get(s3);

     

       599
       599
       -
               		var w4 = stations.get(s4);

     

       600
       600
       -
               		w1.measurement(sb);

     

       601
       601
       -
               		w2.measurement(sb);

     

       602
       602
       -
               		w3.measurement(sb);

     

       603
       603
       -
               		w4.measurement(sb);

     

       604
       604
       -
               	}

     

       572
       572
       +
                   {

     

       573
       573
       +
                       // try to fill teh cpu pipeline as much as possible with

     

       574
       574
       +
                       // independent operations

     

       575
       575
       +
                       int s1 = r1.nextInt(stationCount);

     

       576
       576
       +
                       int s2 = r2.nextInt(stationCount);

     

       577
       577
       +
                       int s3 = r3.nextInt(stationCount);

     

       578
       578
       +
                       int s4 = r4.nextInt(stationCount);

     

       579
       579
       +
                       // get us the ojects one after the other to have the array

     

       580
       580
       +
                       // in our L1 cache and not push it out with other data

     

       581
       581
       +
                       var w1 = stations.get(s1);

     

       582
       582
       +
                       var w2 = stations.get(s2);

     

       583
       583
       +
                       var w3 = stations.get(s3);

     

       584
       584
       +
                       var w4 = stations.get(s4);

     

       585
       585
       +
                       // write our data to our buffer

     

       586
       586
       +
                       w1.measurement(sb);

     

       587
       587
       +
                       w2.measurement(sb);

     

       588
       588
       +
                       w3.measurement(sb);

     

       589
       589
       +
                       w4.measurement(sb);

     

       590
       590
       +
                   }

     

       591
       591
       +
                   {

     

       592
       592
       +
                       int s1 = r1.nextInt(stationCount);

     

       593
       593
       +
                       int s2 = r2.nextInt(stationCount);

     

       594
       594
       +
                       int s3 = r3.nextInt(stationCount);

     

       595
       595
       +
                       int s4 = r4.nextInt(stationCount);

     

       596
       596
       +
                       var w1 = stations.get(s1);

     

       597
       597
       +
                       var w2 = stations.get(s2);

     

       598
       598
       +
                       var w3 = stations.get(s3);

     

       599
       599
       +
                       var w4 = stations.get(s4);

     

       600
       600
       +
                       w1.measurement(sb);

     

       601
       601
       +
                       w2.measurement(sb);

     

       602
       602
       +
                       w3.measurement(sb);

     

       603
       603
       +
                       w4.measurement(sb);

     

       604
       604
       +
                   }

     

       605
       605
        
                   // write the buffer directly, no intermediate string copy

     

       606
       606
        
                   bw.write(sb.data_, 0, sb.length_);

     

       607
       607