Update - Minor edits.

Author: ikelaiah

entries/ikelaiah/src/weatherstation.pas

@@ -53,6 +53,7 @@   TWeatherStation = class
     procedure CreateLookupTemp;
     procedure ReadMeasurements;
     procedure ReadMeasurementsBuf;
+    procedure ReadMeasurementsBufSL;
     procedure ParseStationAndTemp(const line: string);
     procedure AddCityTemperatureLG(const cityName: string; const newTemp: int64);
     procedure SortWeatherStationAndStats;
@@ -328,7 +329,7 @@ procedure TWeatherStation.ReadMeasurements;
   // Open the file for reading
   fileStream := TFileStream.Create(self.fname, fmOpenRead);
   try
-    streamReader := TStreamReader.Create(fileStream, 65536 * 2, False);
+    streamReader := TStreamReader.Create(fileStream, 65536 * 16, False);
     try
       // Read and parse chunks of data until EOF -------------------------------
       while not streamReader.EOF do
@@ -355,7 +356,7 @@ procedure TWeatherStation.ReadMeasurementsBuf;
   index, lineCount: int64;
 begin
 
-  chunksize := 536870912 * 1;
+  chunksize := 4194304 * 1;
 
   // Open the file for reading
   fileStream := TFileStream.Create(self.fname, fmOpenRead);
@@ -432,12 +433,96 @@ procedure TWeatherStation.ReadMeasurementsBuf;
   end;
 end;
 
+procedure TWeatherStation.ReadMeasurementsBufSL;
+var
+  fileStream: TFileStream;
+  strList: TStringList;
+  streamReader: TStreamReader;
+  buffer, trimmedBuffer: TBytes;
+  bytesRead, totalBytesRead, chunkSize, lineBreakPos, chunkIndex,
+  slIndex, lineCount: int64;
+begin
+
+  chunksize := 8192 * 1;
+
+  // Open the file for reading
+  fileStream := TFileStream.Create(self.fname, fmOpenRead);
+  SetLength(buffer, chunkSize);
+  try
+    strList := TStringList.Create;
+    try
+      totalBytesRead := 0;
+      chunkIndex := 0;
+      lineCount := 0;
+
+      // Read and parse chunks of data until EOF
+      while totalBytesRead < fileStream.Size do
+      begin
+        // Read more bytes and keep track on bytes read
+        bytesRead := fileStream.Read(buffer[0], chunkSize);
+        Inc(totalBytesRead, bytesRead);
+
+        // Find the position of the last newline character in the chunk
+        lineBreakPos := BytesRead;
+        while (lineBreakPos > 0) and (Buffer[lineBreakPos - 1] <> Ord(#10)) do
+          Dec(lineBreakPos);
+
+        { Now, must ensure that if the last byte read in the current chunk
+          is not a newline character, the file pointer is moved back to include
+          that byte and any preceding bytes of the partial line in the next
+          chunk's read operation.
+
+          Also, no need to update the BytesRead variable in this context because
+          it represents the actual number of bytes read from the file, including
+          any partial line that may have been included due to moving the file
+          pointer back.
+          Ref: https://www.freepascal.org/docs-html/rtl/classes/tstream.seek.html}
+        if lineBreakPos < bytesRead then
+          fileStream.Seek(-(bytesRead - lineBreakPos), soCurrent);
+        {$IFDEF DEBUG}
+        // Do something with the chunk here
+        // Like counting line
+        for index := 0 to lineBreakPos - 1 do
+          if buffer[index] = Ord(#10) then
+            lineCount := lineCount + 1;
+        {$ENDIF DEBUG}
+
+        // Use TStringList and a sub-TBytes array up to lineBreakPos
+        SetLength(trimmedBuffer, lineBreakPos);
+        // Index 'n' is inclusive, so add 1 to the length
+        Move(buffer[0], trimmedBuffer[0], Length(trimmedBuffer)); // Copy the bytes
+        strList.Clear;
+        strList.Text := ansistring(trimmedBuffer);
+        for slIndex := 0 to strList.Count - 1 do
+          self.ParseStationAndTemp(strList[slIndex]);
+
+        {$IFDEF DEBUG}
+        // Display user feedback
+        WriteLn('Line count: ', IntToStr(lineCount));
+        WriteLn('Chunk ', chunkIndex, ', Total bytes read:', IntToStr(totalBytesRead));
+        {$ENDIF DEBUG}
+
+        {$IFDEF DEBUG}
+        // Increase chunk index - a counter
+        Inc(chunkIndex);
+        {$ENDIF DEBUG}
+      end;
+    finally
+      strList.Free;
+    end;
+  finally
+    // Close the file
+    fileStream.Free;
+  end;
+end;
+
 // The main algorithm
 procedure TWeatherStation.ProcessMeasurements;
 begin
   self.CreateLookupTemp;
   self.ReadMeasurements;
-  //self.ReadMeasurementsBuf;
+  // self.ReadMeasurementsBuf;
+  //self.ReadMeasurementsBufSL;
   self.SortWeatherStationAndStats;
   self.PrintSortedWeatherStationAndStats;
 end;