···

       
107
       
107
       
 
       
    hottest_temp = 99.9

     

       
108
       
108
       
 
       
    station_names_10k_max = random.choices(weather_station_names, k=10_000)

     

       
109
       
109
       
 
       
    batch_size = 10000 # instead of writing line by line to file, process a batch of stations and put it to disk

     

       
110
       
110
       
-
       
    progress_step = max(1, (num_rows_to_create // batch_size) // 100)

     

       
110
       
110
       
+
       
    chunks = num_rows_to_create // batch_size

     

       
111
       
111
       
 
       
    print('Building test data...')

     

       
112
       
112
       
 
       


     

       
113
       
113
       
 
       
    try:

     

       
114
       
114
       
 
       
        with open("../../../data/measurements.txt", 'w') as file:

     

       
115
       
115
       
-
       
            for s in range(0,num_rows_to_create // batch_size):

     

       
115
       
115
       
+
       
            progress = 0

     

       
116
       
116
       
+
       
            for chunk in range(chunks):

     

       
116
       
117
       
 
       
                

     

       
117
       
118
       
 
       
                batch = random.choices(station_names_10k_max, k=batch_size)

     

       
118
       
119
       
 
       
                prepped_deviated_batch = '\n'.join([f"{station};{random.uniform(coldest_temp, hottest_temp):.1f}" for station in batch]) # :.1f should quicker than round on a large scale, because round utilizes mathematical operation

     

       
119
       
120
       
 
       
                file.write(prepped_deviated_batch + '\n')

     

       
120
       
121
       
 
       
                

     

       
121
       
122
       
 
       
                # Update progress bar every 1%

     

       
122
       
122
       
-
       
                if s % progress_step == 0 or s == num_rows_to_create - 1:

     

       
123
       
123
       
-
       
                    sys.stdout.write('\r')

     

       
124
       
124
       
-
       
                    sys.stdout.write("[%-50s] %d%%" % ('=' * int((s + 1) / num_rows_to_create * 50), (s + 1) / num_rows_to_create * 100))

     

       
123
       
123
       
+
       
                if (chunk + 1) * 100 // chunks != progress:

     

       
124
       
124
       
+
       
                    progress = (chunk + 1) * 100 // chunks

     

       
125
       
125
       
+
       
                    bars = '=' * (progress // 2)

     

       
126
       
126
       
+
       
                    sys.stdout.write(f"\r[{bars:<50}] {progress}%")

     

       
125
       
127
       
 
       
                    sys.stdout.flush()

     

       
126
       
128
       
 
       
        sys.stdout.write('\n')

     

       
127
       
129
       
 
       
    except Exception as e: