commit d2639b7ce144ab453e36a36fb6dd3ecf19ed5123 · treybastian.com/1brc

+38

calculate_average_linl33.sh

···

       1
       +
       #!/bin/sh

     

       2
       +
       #

     

       3
       +
       #  Copyright 2023 The original authors

     

       4
       +
       #

     

       5
       +
       #  Licensed under the Apache License, Version 2.0 (the "License");

     

       6
       +
       #  you may not use this file except in compliance with the License.

     

       7
       +
       #  You may obtain a copy of the License at

     

       8
       +
       #

     

       9
       +
       #      http://www.apache.org/licenses/LICENSE-2.0

     

       10
       +
       #

     

       11
       +
       #  Unless required by applicable law or agreed to in writing, software

     

       12
       +
       #  distributed under the License is distributed on an "AS IS" BASIS,

     

       13
       +
       #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

     

       14
       +
       #  See the License for the specific language governing permissions and

     

       15
       +
       #  limitations under the License.

     

       16
       +
       #

     

       17
       +
       

     

       18
       +
       

     

       19
       +
       JAVA_OPTS="-Xrs --enable-preview --add-modules jdk.incubator.vector --enable-native-access=ALL-UNNAMED"

     

       20
       +
       JAVA_OPTS="${JAVA_OPTS} -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions"

     

       21
       +
       JAVA_OPTS="${JAVA_OPTS} -Xms128m -XX:+AlwaysPreTouch -XX:+AlwaysPreTouchStacks -XX:-UseTransparentHugePages"

     

       22
       +
       JAVA_OPTS="${JAVA_OPTS} -XX:-UseCompressedClassPointers -XX:+ForceUnreachable -XX:-CompactStrings"

     

       23
       +
       JAVA_OPTS="${JAVA_OPTS} -XX:CodeEntryAlignment=64 -XX:OptoLoopAlignment=64 -XX:MaxLoopPad=16 -XX:ObjectAlignmentInBytes=64"

     

       24
       +
       JAVA_OPTS="${JAVA_OPTS} -XX:-UseLoopPredicate -XX:LoopStripMiningIter=0 -XX:LoopStripMiningIterShortLoop=0"

     

       25
       +
       JAVA_OPTS="${JAVA_OPTS} -XX:-UseCountedLoopSafepoints -XX:GuaranteedSafepointInterval=0 -XX:AllocatePrefetchStyle=0"

     

       26
       +
       JAVA_OPTS="${JAVA_OPTS} -XX:+TrustFinalNonStaticFields -XX:LockingMode=2 -XX:+UseSystemMemoryBarrier"

     

       27
       +
       JAVA_OPTS="${JAVA_OPTS} -XX:-UseDynamicNumberOfCompilerThreads -XX:-UseDynamicNumberOfGCThreads"

     

       28
       +
       JAVA_OPTS="${JAVA_OPTS} -XX:ArchiveRelocationMode=0 -XX:-UsePerfData -XX:-UseNotificationThread -XX:-CheckIntrinsics"

     

       29
       +
       #JAVA_OPTS="${JAVA_OPTS} -XX:+UseZGC -XX:-ZProactive -XX:+ZCollectionIntervalOnly -XX:ZCollectionInterval=0 -XX:-ZUncommit -XX:-ZBufferStoreBarriers -XX:ZIndexDistributorStrategy=1"

     

       30
       +
       JAVA_OPTS="${JAVA_OPTS} -XX:+UseEpsilonGC -XX:-UseCompressedOops"

     

       31
       +
       #JAVA_OPTS="${JAVA_OPTS} -XX:+UseParallelGC -XX:-UseCompressedOops"

     

       32
       +
       #JAVA_OPTS="${JAVA_OPTS} -XX:+UseG1GC -XX:-UseCompressedOops"

     

       33
       +
       JAVA_OPTS="${JAVA_OPTS} -Djdk.incubator.vector.VECTOR_ACCESS_OOB_CHECK=0 -Djava.lang.invoke.VarHandle.VAR_HANDLE_GUARDS=false -Djava.lang.invoke.MethodHandle.DONT_INLINE_THRESHOLD=-1"

     

       34
       +
       JAVA_OPTS="${JAVA_OPTS} -Dfile.encoding=UTF-8 -Dsun.stdout.encoding=UTF-8 -Dsun.stderr.encoding=UTF-8"

     

       35
       +
       

     

       36
       +
       JAVA_OPTS="${JAVA_OPTS} -Xlog:all=off -Xverify:none -XX:SharedArchiveFile=target/CalculateAverage_linl33_dynamic.jsa"

     

       37
       +
       

     

       38
       +
       MALLOC_ARENA_MAX=1 java ${JAVA_OPTS} --class-path target/average-1.0.0-SNAPSHOT.jar dev.morling.onebrc.CalculateAverage_linl33 2>/dev/null

evaluate.sh

···

       36
        
       MEASUREMENTS_FILE="measurements_1B.txt"

     

       37
        
       RUNS=5

     

       38
        
       DEFAULT_JAVA_VERSION="21.0.1-open"

     

       0
        
       
     

       39
        
       RUN_TIME_LIMIT=300 # seconds

     

       40
        
       

     

       41
        
       TIMEOUT=""

     
···

       115
        
         fi

     

       116
        
       fi

     

       117
        
       

     

       0
        
       
     

       118
        
       print_and_execute java --version

     

       119
        
       print_and_execute ./mvnw --quiet clean verify

     

       120

···

       36
        
       MEASUREMENTS_FILE="measurements_1B.txt"

     

       37
        
       RUNS=5

     

       38
        
       DEFAULT_JAVA_VERSION="21.0.1-open"

     

       39
       +
       : "${BUILD_JAVA_VERSION:=21.0.1-open}"

     

       40
        
       RUN_TIME_LIMIT=300 # seconds

     

       41
        
       

     

       42
        
       TIMEOUT=""

     
···

       116
        
         fi

     

       117
        
       fi

     

       118
        
       

     

       119
       +
       print_and_execute sdk use java $BUILD_JAVA_VERSION

     

       120
        
       print_and_execute java --version

     

       121
        
       print_and_execute ./mvnw --quiet clean verify

     

       122

+33 -1

pom.xml

···

       83
        
                 <artifactId>formatter-maven-plugin</artifactId>

     

       84
        
                 <version>2.16.0</version>

     

       85
        
                 <configuration>

     

       86
       -
                     <configFile>etc/eclipse-formatter-config.xml</configFile>

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       87
        
                 </configuration>

     

       88
        
               </plugin>

     

       89
        
               <plugin>

     
···

       286
        
             <properties>

     

       287
        
               <skipTests>true</skipTests>

     

       288
        
             </properties>

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       289
        
           </profile>

     

       290
        
         </profiles>

     

       291
        
       </project>

···

       83
        
                 <artifactId>formatter-maven-plugin</artifactId>

     

       84
        
                 <version>2.16.0</version>

     

       85
        
                 <configuration>

     

       86
       +
                   <configFile>etc/eclipse-formatter-config.xml</configFile>

     

       87
       +
                   <directories>

     

       88
       +
                     <directory>${project.build.sourceDirectory}</directory>

     

       89
       +
                     <directory>${project.basedir}/src/main/java-22</directory>

     

       90
       +
                   </directories>

     

       91
        
                 </configuration>

     

       92
        
               </plugin>

     

       93
        
               <plugin>

     
···

       290
        
             <properties>

     

       291
        
               <skipTests>true</skipTests>

     

       292
        
             </properties>

     

       293
       +
           </profile>

     

       294
       +
           <profile>

     

       295
       +
             <id>jdk22</id>

     

       296
       +
             <activation>

     

       297
       +
               <jdk>22</jdk>

     

       298
       +
             </activation>

     

       299
       +
             <properties>

     

       300
       +
             </properties>

     

       301
       +
             <build>

     

       302
       +
               <pluginManagement>

     

       303
       +
                 <plugins>

     

       304
       +
                   <plugin>

     

       305
       +
                     <artifactId>maven-compiler-plugin</artifactId>

     

       306
       +
                     <configuration>

     

       307
       +
                       <release>22</release>

     

       308
       +
                       <compileSourceRoots>

     

       309
       +
                         <compileSourceRoot>${project.basedir}/src/main/java-22</compileSourceRoot>

     

       310
       +
                         <!--

     

       311
       +
                           Uncomment the next line to build the entire project with jdk22.

     

       312
       +
                           Currently, some classes fail to compile under jdk22.

     

       313
       +
                         -->

     

       314
       +
                         <!-- <compileSourceRoot>${project.build.sourceDirectory}</compileSourceRoot> -->

     

       315
       +
                       </compileSourceRoots>

     

       316
       +
                     </configuration>

     

       317
       +
                   </plugin>

     

       318
       +
                 </plugins>

     

       319
       +
               </pluginManagement>

     

       320
       +
             </build>

     

       321
        
           </profile>

     

       322
        
         </profiles>

     

       323
        
       </project>

+46

prepare_linl33.sh

···

       1
       +
       #!/bin/bash

     

       2
       +
       #

     

       3
       +
       #  Copyright 2023 The original authors

     

       4
       +
       #

     

       5
       +
       #  Licensed under the Apache License, Version 2.0 (the "License");

     

       6
       +
       #  you may not use this file except in compliance with the License.

     

       7
       +
       #  You may obtain a copy of the License at

     

       8
       +
       #

     

       9
       +
       #      http://www.apache.org/licenses/LICENSE-2.0

     

       10
       +
       #

     

       11
       +
       #  Unless required by applicable law or agreed to in writing, software

     

       12
       +
       #  distributed under the License is distributed on an "AS IS" BASIS,

     

       13
       +
       #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

     

       14
       +
       #  See the License for the specific language governing permissions and

     

       15
       +
       #  limitations under the License.

     

       16
       +
       #

     

       17
       +
       

     

       18
       +
       

     

       19
       +
       source "$HOME/.sdkman/bin/sdkman-init.sh"

     

       20
       +
       # TODO: bump to ea 32 when available

     

       21
       +
       sdk use java 22.ea.31-open 1>&2

     

       22
       +
       

     

       23
       +
       CLASS_NAME="CalculateAverage_linl33"

     

       24
       +
       

     

       25
       +
       JAVA_OPTS="-Xrs --enable-preview --add-modules jdk.incubator.vector --enable-native-access=ALL-UNNAMED"

     

       26
       +
       JAVA_OPTS="${JAVA_OPTS} -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions"

     

       27
       +
       JAVA_OPTS="${JAVA_OPTS} -Xms128m -XX:+AlwaysPreTouch -XX:+AlwaysPreTouchStacks -XX:-UseTransparentHugePages"

     

       28
       +
       JAVA_OPTS="${JAVA_OPTS} -XX:-UseCompressedClassPointers -XX:+ForceUnreachable -XX:-CompactStrings"

     

       29
       +
       JAVA_OPTS="${JAVA_OPTS} -XX:CodeEntryAlignment=64 -XX:OptoLoopAlignment=64 -XX:MaxLoopPad=16 -XX:ObjectAlignmentInBytes=64"

     

       30
       +
       JAVA_OPTS="${JAVA_OPTS} -XX:-UseLoopPredicate -XX:LoopStripMiningIter=0 -XX:LoopStripMiningIterShortLoop=0"

     

       31
       +
       JAVA_OPTS="${JAVA_OPTS} -XX:-UseCountedLoopSafepoints -XX:GuaranteedSafepointInterval=0 -XX:AllocatePrefetchStyle=0"

     

       32
       +
       JAVA_OPTS="${JAVA_OPTS} -XX:+TrustFinalNonStaticFields -XX:LockingMode=2 -XX:+UseSystemMemoryBarrier"

     

       33
       +
       JAVA_OPTS="${JAVA_OPTS} -XX:-UseDynamicNumberOfCompilerThreads -XX:-UseDynamicNumberOfGCThreads"

     

       34
       +
       JAVA_OPTS="${JAVA_OPTS} -XX:ArchiveRelocationMode=0 -XX:-UsePerfData -XX:-UseNotificationThread -XX:-CheckIntrinsics"

     

       35
       +
       #JAVA_OPTS="${JAVA_OPTS} -XX:+UseZGC -XX:-ZProactive -XX:+ZCollectionIntervalOnly -XX:ZCollectionInterval=0 -XX:-ZUncommit -XX:-ZBufferStoreBarriers -XX:ZIndexDistributorStrategy=1"

     

       36
       +
       JAVA_OPTS="${JAVA_OPTS} -XX:+UseEpsilonGC -XX:-UseCompressedOops"

     

       37
       +
       #JAVA_OPTS="${JAVA_OPTS} -XX:+UseParallelGC -XX:-UseCompressedOops"

     

       38
       +
       #JAVA_OPTS="${JAVA_OPTS} -XX:+UseG1GC -XX:-UseCompressedOops"

     

       39
       +
       JAVA_OPTS="${JAVA_OPTS} -Djdk.incubator.vector.VECTOR_ACCESS_OOB_CHECK=0 -Djava.lang.invoke.VarHandle.VAR_HANDLE_GUARDS=false -Djava.lang.invoke.MethodHandle.DONT_INLINE_THRESHOLD=-1"

     

       40
       +
       JAVA_OPTS="${JAVA_OPTS} -Dfile.encoding=UTF-8 -Dsun.stdout.encoding=UTF-8 -Dsun.stderr.encoding=UTF-8"

     

       41
       +
       JAVA_OPTS="${JAVA_OPTS} -Ddev.morling.onebrc.CalculateAverage_linl33.measurementsPath=src/test/resources/samples/measurements-10000-unique-keys.txt"

     

       42
       +
       

     

       43
       +
       # create CDS archive

     

       44
       +
       java ${JAVA_OPTS} -Xshare:off -XX:DumpLoadedClassList=target/${CLASS_NAME}.classlist --class-path target/average-1.0.0-SNAPSHOT.jar dev.morling.onebrc.${CLASS_NAME}

     

       45
       +
       java ${JAVA_OPTS} -Xshare:dump -XX:SharedClassListFile=target/${CLASS_NAME}.classlist -XX:SharedArchiveFile=target/${CLASS_NAME}.jsa --class-path target/average-1.0.0-SNAPSHOT.jar

     

       46
       +
       java ${JAVA_OPTS} -Xshare:on -XX:SharedArchiveFile=target/${CLASS_NAME}.jsa -XX:ArchiveClassesAtExit=target/${CLASS_NAME}_dynamic.jsa --class-path target/average-1.0.0-SNAPSHOT.jar dev.morling.onebrc.${CLASS_NAME}

+520

src/main/java-22/dev/morling/onebrc/CalculateAverage_linl33.java

···

       1
       +
       /*

     

       2
       +
        *  Copyright 2023 The original authors

     

       3
       +
        *

     

       4
       +
        *  Licensed under the Apache License, Version 2.0 (the "License");

     

       5
       +
        *  you may not use this file except in compliance with the License.

     

       6
       +
        *  You may obtain a copy of the License at

     

       7
       +
        *

     

       8
       +
        *      http://www.apache.org/licenses/LICENSE-2.0

     

       9
       +
        *

     

       10
       +
        *  Unless required by applicable law or agreed to in writing, software

     

       11
       +
        *  distributed under the License is distributed on an "AS IS" BASIS,

     

       12
       +
        *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

     

       13
       +
        *  See the License for the specific language governing permissions and

     

       14
       +
        *  limitations under the License.

     

       15
       +
        */

     

       16
       +
       package dev.morling.onebrc;

     

       17
       +
       

     

       18
       +
       import jdk.incubator.vector.ByteVector;

     

       19
       +
       import jdk.incubator.vector.VectorSpecies;

     

       20
       +
       import sun.misc.Unsafe;

     

       21
       +
       

     

       22
       +
       import java.io.IOException;

     

       23
       +
       import java.lang.foreign.*;

     

       24
       +
       import java.lang.invoke.MethodHandle;

     

       25
       +
       import java.nio.ByteOrder;

     

       26
       +
       import java.nio.channels.FileChannel;

     

       27
       +
       import java.nio.charset.StandardCharsets;

     

       28
       +
       import java.nio.file.Paths;

     

       29
       +
       import java.util.Arrays;

     

       30
       +
       import java.util.concurrent.CompletableFuture;

     

       31
       +
       import java.util.concurrent.Executors;

     

       32
       +
       import java.util.stream.IntStream;

     

       33
       +
       

     

       34
       +
       public class CalculateAverage_linl33 {

     

       35
       +
           private static final String FILE_PATH_PROPERTY = "dev.morling.onebrc.CalculateAverage_linl33.measurementsPath";

     

       36
       +
           private static final int WEATHER_STATION_LENGTH_MAX = 100;

     

       37
       +
           private static final long WEATHER_STATION_DISTINCT_MAX = 10_000L;

     

       38
       +
           private static final int N_THREADS = Runtime.getRuntime().availableProcessors();

     

       39
       +
       

     

       40
       +
           private static final MemorySegment ALL = MemorySegment.NULL.reinterpret(Long.MAX_VALUE);

     

       41
       +
           private static final VectorSpecies<Byte> BYTE_SPECIES = ByteVector.SPECIES_PREFERRED;

     

       42
       +
       

     

       43
       +
           private static final Thread.Builder THREAD_BUILDER = Thread

     

       44
       +
                   .ofPlatform()

     

       45
       +
                   .name("1brc-CalculateAverage-", 0)

     

       46
       +
                   .inheritInheritableThreadLocals(false);

     

       47
       +
       

     

       48
       +
           private static final Unsafe UNSAFE;

     

       49
       +
       

     

       50
       +
           static {

     

       51
       +
               if (ByteOrder.nativeOrder() != ByteOrder.LITTLE_ENDIAN) {

     

       52
       +
                   throw new UnsupportedOperationException("Error: BE JVMs are not supported");

     

       53
       +
               }

     

       54
       +
               if ((BYTE_SPECIES.vectorByteSize() & (BYTE_SPECIES.vectorByteSize() - 1)) != 0) {

     

       55
       +
                   throw new UnsupportedOperationException(STR."Unsupported vectorByteSize \{BYTE_SPECIES.vectorByteSize()}");

     

       56
       +
               }

     

       57
       +
       

     

       58
       +
               try {

     

       59
       +
                   var f = Unsafe.class.getDeclaredField("theUnsafe");

     

       60
       +
                   f.setAccessible(true);

     

       61
       +
                   UNSAFE = (Unsafe) f.get(null);

     

       62
       +
               } catch (NoSuchFieldException | IllegalAccessException e) {

     

       63
       +
                   throw new RuntimeException(e);

     

       64
       +
               }

     

       65
       +
           }

     

       66
       +
       

     

       67
       +
           public static void main() throws InterruptedException, IOException {

     

       68
       +
               final var filePath = Paths.get(System.getProperty(FILE_PATH_PROPERTY, "./measurements.txt"));

     

       69
       +
       

     

       70
       +
               try (final var channel = FileChannel.open(filePath)) {

     

       71
       +
                   final var inputMapped = channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size(), Arena.global());

     

       72
       +
       

     

       73
       +
                   final var chunkBounds = calcChunkBounds(inputMapped.address(), inputMapped.byteSize());

     

       74
       +
                   final var maps = new SparseMap[N_THREADS];

     

       75
       +
       

     

       76
       +
                   try (final var threadPool = Executors.newFixedThreadPool(N_THREADS, THREAD_BUILDER.factory());

     

       77
       +
                           final var singleThreadExecutor = Executors.newSingleThreadExecutor(Thread.ofVirtual().factory())) {

     

       78
       +
                       final var rootTask = CompletableFuture.runAsync(new CalculateAverageTask(maps, chunkBounds, 0), threadPool);

     

       79
       +
       

     

       80
       +
                       final var futures = IntStream

     

       81
       +
                               .range(1, N_THREADS)

     

       82
       +
                               .mapToObj(t -> CompletableFuture

     

       83
       +
                                       .runAsync(new CalculateAverageTask(maps, chunkBounds, t), threadPool)

     

       84
       +
                                       .runAfterBothAsync(rootTask, () -> maps[0].merge(maps[t]), singleThreadExecutor))

     

       85
       +
                               .toArray(CompletableFuture[]::new);

     

       86
       +
       

     

       87
       +
                       CompletableFuture.allOf(futures).join();

     

       88
       +
                   }

     

       89
       +
       

     

       90
       +
                   printSorted(maps[0]);

     

       91
       +
               }

     

       92
       +
           }

     

       93
       +
       

     

       94
       +
           private static long[] calcChunkBounds(final long mappedAddr, final long fileSizeBytes) {

     

       95
       +
               final var chunkBounds = new long[N_THREADS + 1];

     

       96
       +
               chunkBounds[0] = mappedAddr;

     

       97
       +
               chunkBounds[chunkBounds.length - 1] = mappedAddr + fileSizeBytes;

     

       98
       +
       

     

       99
       +
               final var chunkSize = (fileSizeBytes / N_THREADS) & -CalculateAverageTask.BATCH_SIZE_BYTES;

     

       100
       +
               for (int i = 1; i < chunkBounds.length - 1; i++) {

     

       101
       +
                   chunkBounds[i] = chunkBounds[i - 1] + chunkSize;

     

       102
       +
               }

     

       103
       +
       

     

       104
       +
               return chunkBounds;

     

       105
       +
           }

     

       106
       +
       

     

       107
       +
           private static void printSorted(final SparseMap temperatureMeasurements) {

     

       108
       +
               final var weatherStations = new AggregatedMeasurement[(int) temperatureMeasurements.size];

     

       109
       +
               final var nameBuffer = new byte[WEATHER_STATION_LENGTH_MAX];

     

       110
       +
               var offset = temperatureMeasurements.denseAddress;

     

       111
       +
               for (int i = 0; i < weatherStations.length; i++, offset += SparseMap.DATA_SCALE * Long.BYTES) {

     

       112
       +
                   final var nameAddr = UNSAFE.getLong(offset);

     

       113
       +
                   final var nameLength = UNSAFE.getInt(offset + Integer.BYTES * 7);

     

       114
       +
                   MemorySegment.copy(ALL, ValueLayout.JAVA_BYTE, nameAddr, nameBuffer, 0, nameLength);

     

       115
       +
                   final var nameStr = new String(nameBuffer, 0, nameLength, StandardCharsets.UTF_8);

     

       116
       +
                   weatherStations[i] = new AggregatedMeasurement(nameStr, i);

     

       117
       +
               }

     

       118
       +
       

     

       119
       +
               Arrays.sort(weatherStations);

     

       120
       +
       

     

       121
       +
               System.out.print('{');

     

       122
       +
               for (int i = 0; i < weatherStations.length - 1; i++) {

     

       123
       +
                   printAggMeasurement(weatherStations[i], temperatureMeasurements);

     

       124
       +
                   System.out.print(',');

     

       125
       +
                   System.out.print(' ');

     

       126
       +
               }

     

       127
       +
               printAggMeasurement(weatherStations[weatherStations.length - 1], temperatureMeasurements);

     

       128
       +
               System.out.println('}');

     

       129
       +
           }

     

       130
       +
       

     

       131
       +
           private static void printAggMeasurement(final AggregatedMeasurement aggMeasurement,

     

       132
       +
                                                   final SparseMap temperatureMeasurements) {

     

       133
       +
               final var offset = temperatureMeasurements.denseAddress + SparseMap.DATA_SCALE * Long.BYTES * aggMeasurement.id();

     

       134
       +
       

     

       135
       +
               // name

     

       136
       +
               System.out.print(aggMeasurement.name());

     

       137
       +
               System.out.print('=');

     

       138
       +
       

     

       139
       +
               // min

     

       140
       +
               printAsDouble(offset + Integer.BYTES * 5);

     

       141
       +
               System.out.print('/');

     

       142
       +
       

     

       143
       +
               // mean

     

       144
       +
               final double total = UNSAFE.getLong(offset + Integer.BYTES * 2);

     

       145
       +
               final var count = UNSAFE.getInt(offset + Integer.BYTES * 4);

     

       146
       +
               System.out.print(round(total / count / 10d));

     

       147
       +
               System.out.print('/');

     

       148
       +
       

     

       149
       +
               // max

     

       150
       +
               printAsDouble(offset + Integer.BYTES * 6);

     

       151
       +
           }

     

       152
       +
       

     

       153
       +
           private static void printAsDouble(final long addr) {

     

       154
       +
               final var val = (double) UNSAFE.getInt(addr);

     

       155
       +
               System.out.print(val / 10d);

     

       156
       +
           }

     

       157
       +
       

     

       158
       +
           private static double round(final double d) {

     

       159
       +
               return Math.round(d * 10d) / 10d;

     

       160
       +
           }

     

       161
       +
       

     

       162
       +
           private static class CalculateAverageTask implements Runnable {

     

       163
       +
               public static final int BATCH_SIZE_BYTES = BYTE_SPECIES.vectorByteSize();

     

       164
       +
       

     

       165
       +
               private final SparseMap[] maps;

     

       166
       +
               private final long[] chunkBounds;

     

       167
       +
               private final long chunkStart;

     

       168
       +
               private final long chunkEnd;

     

       169
       +
               private final int t;

     

       170
       +
       

     

       171
       +
               private SparseMap map;

     

       172
       +
       

     

       173
       +
               public CalculateAverageTask(SparseMap[] maps, long[] chunkBounds, int t) {

     

       174
       +
                   this.maps = maps;

     

       175
       +
                   this.chunkBounds = chunkBounds;

     

       176
       +
                   this.chunkStart = chunkBounds[t];

     

       177
       +
                   this.chunkEnd = chunkBounds[t + 1];

     

       178
       +
                   this.t = t;

     

       179
       +
               }

     

       180
       +
       

     

       181
       +
               @Override

     

       182
       +
               public void run() {

     

       183
       +
                   this.maps[this.t] = new SparseMap();

     

       184
       +
                   this.map = this.maps[this.t];

     

       185
       +
       

     

       186
       +
                   var lineStart = this.chunkBounds[0];

     

       187
       +
                   // walk back to find the previous '\n' and use it as lineStart

     

       188
       +
                   for (long i = this.chunkStart - 1; i > this.chunkBounds[0]; i--) {

     

       189
       +
                       if (UNSAFE.getByte(i) == (byte) '\n') {

     

       190
       +
                           lineStart = i + 1L;

     

       191
       +
                           break;

     

       192
       +
                       }

     

       193
       +
                   }

     

       194
       +
       

     

       195
       +
                   final var vectorLimit = this.chunkStart + ((this.chunkEnd - this.chunkStart) & -BYTE_SPECIES.vectorByteSize());

     

       196
       +
                   for (long i = this.chunkStart; i < vectorLimit; i += BYTE_SPECIES.vectorByteSize()) {

     

       197
       +
                       var lfMask = ByteVector.fromMemorySegment(BYTE_SPECIES, ALL, i, ByteOrder.nativeOrder())

     

       198
       +
                               .eq((byte) '\n')

     

       199
       +
                               .toLong();

     

       200
       +
       

     

       201
       +
                       final var lfCount = Long.bitCount(lfMask);

     

       202
       +
                       for (int j = 0; j < lfCount; j++) {

     

       203
       +
                           final var lfPosRelative = Long.numberOfTrailingZeros(lfMask);

     

       204
       +
                           final var lfAddress = i + lfPosRelative;

     

       205
       +
                           processLine(lineStart, lfAddress);

     

       206
       +
       

     

       207
       +
                           lineStart = lfAddress + 1L;

     

       208
       +
                           // unset the lowest set bit, should compile to BLSR

     

       209
       +
                           lfMask &= lfMask - 1L;

     

       210
       +
                       }

     

       211
       +
                   }

     

       212
       +
       

     

       213
       +
                   if (vectorLimit != this.chunkEnd) {

     

       214
       +
                       processTrailingBytes(lineStart, vectorLimit, this.chunkEnd);

     

       215
       +
                   }

     

       216
       +
               }

     

       217
       +
       

     

       218
       +
               private void processTrailingBytes(long lineStart,

     

       219
       +
                                                 final long start,

     

       220
       +
                                                 final long end) {

     

       221
       +
                   for (long i = start; i < end; i++) {

     

       222
       +
                       final var b = UNSAFE.getByte(i);

     

       223
       +
                       if (b != (byte) '\n') {

     

       224
       +
                           continue;

     

       225
       +
                       }

     

       226
       +
       

     

       227
       +
                       processLine(lineStart, i);

     

       228
       +
                       lineStart = i + 1;

     

       229
       +
                   }

     

       230
       +
               }

     

       231
       +
       

     

       232
       +
               private void processLine(final long lineStart, final long lfAddress) {

     

       233
       +
                   // read 5 bytes before '\n'

     

       234
       +
                   // the temperature is formatted to 1 decimal place

     

       235
       +
                   // therefore the shortest temperature value is 0.0

     

       236
       +
                   // so there are always at least 5 bytes between the location name and '\n'

     

       237
       +
                   final var trailing5Bytes = UNSAFE.getLong(lfAddress - 5);

     

       238
       +
                   final int trailingDWordRaw = (int) (trailing5Bytes >>> 8);

     

       239
       +
       

     

       240
       +
                   // select the low nibble for each byte, '0'-'9' -> 0-9, ';' -> 11, '-' -> 13

     

       241
       +
                   final var trailingDWordLowNibble = trailingDWordRaw & 0x0f_0f_0f_0f;

     

       242
       +
                   // parse the 2 digits around the decimal point (note that these 2 digits must be present)

     

       243
       +
                   final var trailingDigitsParsed = (trailingDWordLowNibble * 0x00_0a_00_01) >>> 24;

     

       244
       +
       

     

       245
       +
                   // this byte must be ('-' & 0xf), (';' & 0xf), or a valid digit (0-9)

     

       246
       +
                   final var secondHighestByte = trailingDWordLowNibble & 0xf;

     

       247
       +
       

     

       248
       +
                   var temperature = trailingDigitsParsed;

     

       249
       +
                   var lineLength = lfAddress - lineStart - 4;

     

       250
       +
       

     

       251
       +
                   if (secondHighestByte > 9) {

     

       252
       +
                       if (secondHighestByte == ('-' & 0xf)) {

     

       253
       +
                           lineLength--;

     

       254
       +
                           temperature = -temperature;

     

       255
       +
                       }

     

       256
       +
                   }

     

       257
       +
                   else {

     

       258
       +
                       lineLength--;

     

       259
       +
                       temperature += secondHighestByte * 100;

     

       260
       +
       

     

       261
       +
                       final var isNegative = (trailing5Bytes & 0xffL) == '-';

     

       262
       +
                       if (isNegative) {

     

       263
       +
                           lineLength--;

     

       264
       +
                           temperature = -temperature;

     

       265
       +
                       }

     

       266
       +
                   }

     

       267
       +
       

     

       268
       +
                   this.map.putEntry(lineStart, (int) lineLength, temperature);

     

       269
       +
               }

     

       270
       +
           }

     

       271
       +
       

     

       272
       +
           /**

     

       273
       +
            * Open addressing, linear probing hash map backed by off-heap memory

     

       274
       +
            */

     

       275
       +
           private static class SparseMap {

     

       276
       +
               private static final int TRUNCATED_HASH_BITS = 26;

     

       277
       +
               // max # of unique keys

     

       278
       +
               private static final long DENSE_SIZE = WEATHER_STATION_DISTINCT_MAX;

     

       279
       +
               // max hash code (exclusive)

     

       280
       +
               private static final long SPARSE_SIZE = 1L << (TRUNCATED_HASH_BITS + 1);

     

       281
       +
               private static final long DATA_SCALE = 4;

     

       282
       +
       

     

       283
       +
               public final long sparseAddress;

     

       284
       +
               public final long denseAddress;

     

       285
       +
               public long size;

     

       286
       +
       

     

       287
       +
               public SparseMap() {

     

       288
       +
                   var arena = new MallocArena(Arena.global());

     

       289
       +
                   var callocArena = new CallocArena(Arena.global());

     

       290
       +
       

     

       291
       +
                   this.size = 0L;

     

       292
       +
       

     

       293
       +
                   final var sparse = callocArena.allocate(ValueLayout.JAVA_LONG, SPARSE_SIZE);

     

       294
       +
                   this.sparseAddress = (sparse.address() + MallocArena.MAX_ALIGN) & -MallocArena.MAX_ALIGN;

     

       295
       +
       

     

       296
       +
                   final var dense = arena.allocate(ValueLayout.JAVA_LONG, DENSE_SIZE * DATA_SCALE);

     

       297
       +
                   this.denseAddress = (dense.address() + MallocArena.MAX_ALIGN) & -MallocArena.MAX_ALIGN;

     

       298
       +
               }

     

       299
       +
       

     

       300
       +
               public void putEntry(final long keyAddress, final int keyLength, final int value) {

     

       301
       +
                   final var hash = hash(keyAddress, keyLength);

     

       302
       +
                   this.putEntryInternal(hash, keyAddress, keyLength, value, 1, value, value);

     

       303
       +
               }

     

       304
       +
       

     

       305
       +
               private void putEntryInternal(final long hash,

     

       306
       +
                                             final long keyAddress,

     

       307
       +
                                             final int keyLength,

     

       308
       +
                                             final long temperature,

     

       309
       +
                                             final int count,

     

       310
       +
                                             final int temperatureMin,

     

       311
       +
                                             final int temperatureMax) {

     

       312
       +
                   final var sparseOffset = this.sparseAddress + truncateHash(hash) * Long.BYTES;

     

       313
       +
       

     

       314
       +
                   for (long n = 0, sparseLinearOffset = sparseOffset; n < WEATHER_STATION_DISTINCT_MAX; n++, sparseLinearOffset += Long.BYTES) {

     

       315
       +
                       final var denseOffset = UNSAFE.getLong(sparseLinearOffset);

     

       316
       +
                       if (denseOffset == 0L) {

     

       317
       +
                           this.add(sparseLinearOffset, keyAddress, keyLength, temperature, count, temperatureMin, temperatureMax);

     

       318
       +
                           this.size++;

     

       319
       +
                           return;

     

       320
       +
                       }

     

       321
       +
       

     

       322
       +
                       if (isCollision(keyAddress, keyLength, denseOffset)) {

     

       323
       +
                           continue;

     

       324
       +
                       }

     

       325
       +
       

     

       326
       +
                       final var currTotal = UNSAFE.getLong(denseOffset + Integer.BYTES * 2);

     

       327
       +
                       UNSAFE.putLong(denseOffset + Integer.BYTES * 2, currTotal + temperature); // total

     

       328
       +
       

     

       329
       +
                       final var currCount = UNSAFE.getInt(denseOffset + Integer.BYTES * 4);

     

       330
       +
                       UNSAFE.putInt(denseOffset + Integer.BYTES * 4, currCount + count); // count

     

       331
       +
       

     

       332
       +
                       final var currMin = UNSAFE.getInt(denseOffset + Integer.BYTES * 5);

     

       333
       +
                       if (temperatureMin < currMin) {

     

       334
       +
                           UNSAFE.putInt(denseOffset + Integer.BYTES * 5, temperatureMin); // min

     

       335
       +
                       }

     

       336
       +
       

     

       337
       +
                       final var currMax = UNSAFE.getInt(denseOffset + Integer.BYTES * 6);

     

       338
       +
                       if (temperatureMax > currMax) {

     

       339
       +
                           UNSAFE.putInt(denseOffset + Integer.BYTES * 6, temperatureMax); // max

     

       340
       +
                       }

     

       341
       +
       

     

       342
       +
                       return;

     

       343
       +
                   }

     

       344
       +
               }

     

       345
       +
       

     

       346
       +
               public void merge(final SparseMap other) {

     

       347
       +
                   final var otherSize = other.size;

     

       348
       +
                   for (long i = 0, offset = other.denseAddress; i < otherSize; i++, offset += DATA_SCALE * Long.BYTES) {

     

       349
       +
                       final var keyAddress = UNSAFE.getLong(offset);

     

       350
       +
                       final var keyLength = UNSAFE.getInt(offset + Integer.BYTES * 7);

     

       351
       +
                       final var hash = hash(keyAddress, keyLength);

     

       352
       +
       

     

       353
       +
                       this.putEntryInternal(

     

       354
       +
                               hash,

     

       355
       +
                               keyAddress,

     

       356
       +
                               keyLength,

     

       357
       +
                               UNSAFE.getLong(offset + Integer.BYTES * 2),

     

       358
       +
                               UNSAFE.getInt(offset + Integer.BYTES * 4),

     

       359
       +
                               UNSAFE.getInt(offset + Integer.BYTES * 5),

     

       360
       +
                               UNSAFE.getInt(offset + Integer.BYTES * 6));

     

       361
       +
                   }

     

       362
       +
               }

     

       363
       +
       

     

       364
       +
               private void add(final long sparseOffset,

     

       365
       +
                                final long keyAddress,

     

       366
       +
                                final int keyLength,

     

       367
       +
                                final long temperature,

     

       368
       +
                                final int count,

     

       369
       +
                                final int temperatureMin,

     

       370
       +
                                final int temperatureMax) {

     

       371
       +
                   // new entry, initialize sparse and dense

     

       372
       +
                   final var denseOffset = this.denseAddress + this.size * DATA_SCALE * Long.BYTES;

     

       373
       +
                   UNSAFE.putLong(sparseOffset, denseOffset);

     

       374
       +
       

     

       375
       +
                   UNSAFE.putLong(denseOffset, keyAddress);

     

       376
       +
                   UNSAFE.putLong(denseOffset + Integer.BYTES * 2, temperature);

     

       377
       +
                   UNSAFE.putInt(denseOffset + Integer.BYTES * 4, count);

     

       378
       +
                   UNSAFE.putInt(denseOffset + Integer.BYTES * 5, temperatureMin);

     

       379
       +
                   UNSAFE.putInt(denseOffset + Integer.BYTES * 6, temperatureMax);

     

       380
       +
                   UNSAFE.putInt(denseOffset + Integer.BYTES * 7, keyLength);

     

       381
       +
               }

     

       382
       +
       

     

       383
       +
               private static boolean isCollision(final long keyAddress, final int keyLength, final long denseOffset) {

     

       384
       +
                   // key length compare is unnecessary

     

       385
       +
       

     

       386
       +
                   final var entryKeyAddress = UNSAFE.getLong(denseOffset);

     

       387
       +
                   return mismatch(keyAddress, entryKeyAddress, keyLength);

     

       388
       +
               }

     

       389
       +
       

     

       390
       +
               private static boolean mismatch(final long leftAddr, final long rightAddr, final int length) {

     

       391
       +
                   // key length compare is unnecessary

     

       392
       +
                   // strings compared through delimiter byte ';'

     

       393
       +
       

     

       394
       +
                   final var loopBound = length >= (BYTE_SPECIES.vectorByteSize() - 1) ? ((length + 1) & -BYTE_SPECIES.vectorByteSize()) : 0;

     

       395
       +
                   for (long i = 0; i < loopBound; i += BYTE_SPECIES.vectorByteSize()) {

     

       396
       +
                       final var l = ByteVector.fromMemorySegment(BYTE_SPECIES, ALL, leftAddr + i, ByteOrder.nativeOrder());

     

       397
       +
                       final var r = ByteVector.fromMemorySegment(BYTE_SPECIES, ALL, rightAddr + i, ByteOrder.nativeOrder());

     

       398
       +
                       if (!l.eq(r).allTrue()) {

     

       399
       +
                           return true;

     

       400
       +
                       }

     

       401
       +
                   }

     

       402
       +
       

     

       403
       +
                   final var l = ByteVector.fromMemorySegment(BYTE_SPECIES, ALL, leftAddr + loopBound, ByteOrder.nativeOrder());

     

       404
       +
                   final var r = ByteVector.fromMemorySegment(BYTE_SPECIES, ALL, rightAddr + loopBound, ByteOrder.nativeOrder());

     

       405
       +
                   final var eqMask = l.eq(r).toLong();

     

       406
       +
       

     

       407
       +
                   // LE compare to add 1 to length

     

       408
       +
                   return Long.numberOfTrailingZeros(~eqMask) <= (length - loopBound);

     

       409
       +
                   // to support platforms without TZCNT, the check can be replaced with

     

       410
       +
                   // a comparison to lowestZero = ~eqMask & (eqMask + 1)

     

       411
       +
               }

     

       412
       +
       

     

       413
       +
               // Use the leading and trailing few bytes as hash

     

       414
       +
               // this performs better than computing a good hash

     

       415
       +
               private static long hash(final long keyAddress, final int keyLength) {

     

       416
       +
                   final var leadingQWord = UNSAFE.getLong(keyAddress);

     

       417
       +
                   // the constant is the 64 bit FNV-1 offset basis

     

       418
       +
                   final var hash = -3750763034362895579L ^ leadingQWord;

     

       419
       +
                   if (keyLength < Integer.BYTES) {

     

       420
       +
                       // the key is at least 2 bytes (if you count the delimiter)

     

       421
       +
                       return hash & 0xffffL;

     

       422
       +
                   }

     

       423
       +
                   else {

     

       424
       +
                       final var trailingDWord = UNSAFE.getLong(keyAddress + keyLength - Integer.BYTES) & 0xffffffffL;

     

       425
       +
                       // only the lower dword in hash is guaranteed to exist so shift left 32

     

       426
       +
                       return (hash << Integer.SIZE) ^ trailingDWord;

     

       427
       +
                   }

     

       428
       +
               }

     

       429
       +
       

     

       430
       +
               private static long truncateHash(final long hash) {

     

       431
       +
                   return ((hash >>> TRUNCATED_HASH_BITS) ^ hash) & ((1L << TRUNCATED_HASH_BITS) - 1L);

     

       432
       +
               }

     

       433
       +
           }

     

       434
       +
       

     

       435
       +
           private static class MallocArena implements Arena {

     

       436
       +
               public static final long MAX_ALIGN = 1L << 21;

     

       437
       +
       

     

       438
       +
               protected static final Linker LINKER = Linker.nativeLinker();

     

       439
       +
               protected static final AddressLayout C_POINTER = (AddressLayout) LINKER.canonicalLayouts().get("void*");

     

       440
       +
               protected static final ValueLayout C_SIZE_T = (ValueLayout) LINKER.canonicalLayouts().get("size_t");

     

       441
       +
               private static final MethodHandle MALLOC = LINKER.downcallHandle(

     

       442
       +
                       LINKER.defaultLookup().find("malloc").orElseThrow(),

     

       443
       +
                       FunctionDescriptor.of(C_POINTER, C_SIZE_T),

     

       444
       +
                       Linker.Option.critical(false));

     

       445
       +
               private static final MethodHandle FREE = LINKER.downcallHandle(

     

       446
       +
                       LINKER.defaultLookup().find("free").orElseThrow(),

     

       447
       +
                       FunctionDescriptor.ofVoid(C_POINTER),

     

       448
       +
                       Linker.Option.critical(false));

     

       449
       +
               protected static final MethodHandle CALLOC = LINKER.downcallHandle(

     

       450
       +
                       LINKER.defaultLookup().find("calloc").orElseThrow(),

     

       451
       +
                       FunctionDescriptor.of(C_POINTER, C_SIZE_T, C_SIZE_T),

     

       452
       +
                       Linker.Option.critical(false));

     

       453
       +
       

     

       454
       +
               private final Arena arena;

     

       455
       +
       

     

       456
       +
               public MallocArena(Arena arena) {

     

       457
       +
                   this.arena = arena;

     

       458
       +
               }

     

       459
       +
       

     

       460
       +
               @Override

     

       461
       +
               public MemorySegment allocate(final long byteSize, final long byteAlignment) {

     

       462
       +
                   return malloc(byteSize + MAX_ALIGN).reinterpret(this, MallocArena::free);

     

       463
       +
               }

     

       464
       +
       

     

       465
       +
               @Override

     

       466
       +
               public MemorySegment.Scope scope() {

     

       467
       +
                   return arena.scope();

     

       468
       +
               }

     

       469
       +
       

     

       470
       +
               @Override

     

       471
       +
               public void close() {

     

       472
       +
                   arena.close();

     

       473
       +
               }

     

       474
       +
       

     

       475
       +
               private static MemorySegment malloc(final long byteSize) {

     

       476
       +
                   try {

     

       477
       +
                       return ((MemorySegment) MALLOC.invokeExact(byteSize)).reinterpret(byteSize);

     

       478
       +
                   }

     

       479
       +
                   catch (Throwable e) {

     

       480
       +
                       throw new RuntimeException(e);

     

       481
       +
                   }

     

       482
       +
               }

     

       483
       +
       

     

       484
       +
               protected static void free(final MemorySegment address) {

     

       485
       +
                   try {

     

       486
       +
                       FREE.invokeExact(address);

     

       487
       +
                   }

     

       488
       +
                   catch (Throwable e) {

     

       489
       +
                       throw new RuntimeException(e);

     

       490
       +
                   }

     

       491
       +
               }

     

       492
       +
           }

     

       493
       +
       

     

       494
       +
           private static class CallocArena extends MallocArena {

     

       495
       +
               public CallocArena(Arena arena) {

     

       496
       +
                   super(arena);

     

       497
       +
               }

     

       498
       +
       

     

       499
       +
               @Override

     

       500
       +
               public MemorySegment allocate(final long byteSize, final long byteAlignment) {

     

       501
       +
                   return calloc(byteSize + MAX_ALIGN).reinterpret(this, MallocArena::free);

     

       502
       +
               }

     

       503
       +
       

     

       504
       +
               private static MemorySegment calloc(final long byteSize) {

     

       505
       +
                   try {

     

       506
       +
                       return ((MemorySegment) MallocArena.CALLOC.invokeExact(1L, byteSize)).reinterpret(byteSize);

     

       507
       +
                   }

     

       508
       +
                   catch (Throwable e) {

     

       509
       +
                       throw new RuntimeException(e);

     

       510
       +
                   }

     

       511
       +
               }

     

       512
       +
           }

     

       513
       +
       

     

       514
       +
           private record AggregatedMeasurement(String name, long id) implements Comparable<AggregatedMeasurement> {

     

       515
       +
       

     

       516
       +
           @Override

     

       517
       +
           public int compareTo(final AggregatedMeasurement other) {

     

       518
       +
               return name.compareTo(other.name);

     

       519
       +
           }

     

       520
       +
       }}