Win32 compatability adjustments. One performance test fails during shutdown as python fails to delete a stream in time it seems, so the file cannot be deleted as it is still opened by someone. This is madness, raising the question how badly python truly fails to call the destructors of objects in order to allow them to release their resources

Author: Byron

db/loose.py

@@ -30,6 +30,8 @@
 		exists,
 		chmod,
 		isdir,
+		isfile,
+		remove,
 		mkdir,
 		rename,
 		dirname,
@@ -60,6 +62,12 @@ class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW):
 	# chunks in which data will be copied between streams
 	stream_chunk_size = chunk_size
 
+	# On windows we need to keep it writable, otherwise it cannot be removed
+	# either
+	new_objects_mode = 0444
+	if os.name == 'nt':
+		new_objects_mode = 0644
+			
 
 	def __init__(self, root_path):
 		super(LooseObjectDB, self).__init__(root_path)
@@ -199,11 +207,15 @@ def store(self, istream):
 			if not isdir(obj_dir):
 				mkdir(obj_dir)
 			# END handle destination directory
+			# rename onto existing doesn't work on windows
+			if os.name == 'nt' and isfile(obj_path):
+				remove(obj_path)
+			# END handle win322
 			rename(tmp_path, obj_path)
 
 			# make sure its readable for all ! It started out as rw-- tmp file
-			# but needs to be rrr
-			chmod(obj_path, 0444)
+			# but needs to be rwrr
+			chmod(obj_path, self.new_objects_mode)
 		# END handle dry_run
 
 		istream.binsha = hex_to_bin(hexsha)

test/lib.py

@@ -19,6 +19,7 @@
 import tempfile
 import shutil
 import os
+import gc
 
 
 #{ Bases
@@ -44,6 +45,11 @@ def wrapper(self):
 				print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path)
 				raise
 		finally:
+			# Need to collect here to be sure all handles have been closed. It appears
+			# a windows-only issue. In fact things should be deleted, as well as 
+			# memory maps closed, once objects go out of scope. For some reason
+			# though this is not the case here unless we collect explicitly.
+			gc.collect()
 			shutil.rmtree(path)
 		# END handle exception
 	# END wrapper

test/test_example.py

@@ -21,6 +21,9 @@ def test_base(self):
 			assert len(ostream.read()) == ostream.size
 			assert ldb.has_object(oinfo.binsha)
 		# END for each sha in database
+		# assure we close all files
+		del(ostream)
+		del(oinfo)
 
 		data = "my data"
 		istream = IStream("blob", len(data), StringIO(data))

test/test_stream.py

@@ -19,6 +19,7 @@
 	str_blob_type
 	)
 
+import time
 import tempfile
 import os
 
@@ -125,12 +126,14 @@ def test_compressed_writer(self):
 			# for now, just a single write, code doesn't care about chunking
 			assert len(data) == ostream.write(data)
 			ostream.close()
+		
 			# its closed already
 			self.failUnlessRaises(OSError, os.close, fd)
 
 			# read everything back, compare to data we zip
-			fd = os.open(path, os.O_RDONLY)
+			fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0))
 			written_data = os.read(fd, os.path.getsize(path))
+			assert len(written_data) == os.path.getsize(path)
 			os.close(fd)
 			assert written_data == zlib.compress(data, 1)	# best speed
 

util.py

@@ -58,12 +58,14 @@ def unpack_from(fmt, data, offset=0):
 isdir = os.path.isdir
 isfile = os.path.isfile
 rename = os.rename
+remove = os.remove
 dirname = os.path.dirname
 basename = os.path.basename
 join = os.path.join
 read = os.read
 write = os.write
 close = os.close
+fsync = os.fsync
 
 # constants
 NULL_HEX_SHA = "0"*40
@@ -128,7 +130,7 @@ def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0):
 	:note: for now we don't try to use O_NOATIME directly as the right value needs to be 
 		shared per database in fact. It only makes a real difference for loose object 
 		databases anyway, and they use it with the help of the ``flags`` parameter"""
-	fd = os.open(filepath, os.O_RDONLY|flags)
+	fd = os.open(filepath, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags)
 	try:
 		return file_contents_ro(fd, stream, allow_mmap)
 	finally:
@@ -300,7 +302,9 @@ def _end_writing(self, successful=True):
 			os.rename(lockfile, self._filepath)
 
 			# assure others can at least read the file - the tmpfile left it at rw--
-			chmod(self._filepath, 0444)
+			# We may also write that file, on windows that boils down to a remove-
+			# protection as well
+			chmod(self._filepath, 0644)
 		else:
 			# just delete the file so far, we failed
 			os.remove(lockfile)