Hi there guys, latelly I have been busy as hell, but wanted to take some time to publish a simple code I created and helps me a lot when I want a quick way to send/receive binary data in a python server-client application.

Since sending binary information with python requires you to use the ctypes and the struct module, this can get a bit complicated. So I created this two helper classes, one is used to send information (Message) and the other is used to parse information (BufferHandler). The idea behind those codes are to provide a way that any python programmer can send/receive binary data with no knowledge of the struct and the ctypes module.

Warning: this is going to be a very large post.

Here are the codes:

First, the constants.py, that is used to keep the constant codes used by ctypes and their sizes.

# constants.py
class FormatConstants:
	UNSIGNED_SHORT_INT_CODE = 'H'
	UNSIGNED_SHORT_INT_SIZE = 2
	CHAR_CODE = 'c'
	CHAR_SIZE = 1
	SHORT_INT_CODE = 'h'
	SHORT_INT_SIZE = 2
	INT_CODE = 'i'
	INT_SIZE = 4
	UNSIGNED_INT_CODE = 'I'
	UNSIGNED_INT_SIZE = 4
	LONG_INT_CODE = 'l'
	LONG_INT_SIZE = 4
	UNSIGNED_LONG_INT_CODE = 'L'
	UNSIGNED_LONG_INT_SIZE = 4
	LONG_LONG_INT_CODE = 'q'
	LONG_LONG_INT_SIZE = 8
	UNSIGNED_LONG_LONG_INT_CODE = 'Q'
	UNSIGNED_LONG_LONG_INT_SIZE = 8
	FLOAT_CODE = 'f'
	FLOAT_SIZE = 4
	DOUBLE_CODE = 'd'
	DOUBLE_SIZE = 8
	STRING_CODE = 's'

Then the buffer handler, used to parse information received:

# buffer_handler.py

from struct import unpack_from
from constants import FormatConstants
from ctypes import create_string_buffer

class BufferHandlerException(Exception):
	pass

class BufferHandler:

	def __init__(self, size):

		self.currentBuffer = create_string_buffer(size)
		self.originalBufferSize = size
		self.currentBufferSize = size
		self.currentBufferOffset = 0

	def getBuffer(self):
		self.currentBufferOffset = 0
		self.currentBufferSize = self.originalBufferSize
		return self.currentBuffer

        def getOriginalSize(self):
                return self.originalBufferSize

	def setLimit(self, limit):
		self.currentBufferSize = limit

	def __checkValidity(self, size):
		if (self.currentBuffer == None):
			raise BufferHandlerException("Buffer is invalid.")

		if (self.currentBufferOffset + size > self.currentBufferSize):
			raise BufferHandlerException("Buffer is too small for the requested data.")

	def __processBufferRequest(self, formatCode, formatSize):
		self.__checkValidity(formatSize)
		value = unpack_from(formatCode, self.currentBuffer, self.currentBufferOffset)
		self.currentBufferOffset += formatSize
		return value[0]

	def getUnsignedShortInt(self):
		return self.__processBufferRequest(FormatConstants.UNSIGNED_SHORT_INT_CODE,
			FormatConstants.UNSIGNED_SHORT_INT_SIZE)

	def getChar(self):
		return self.__processBufferRequest(FormatConstants.CHAR_CODE,
			FormatConstants.CHAR_SIZE)


	def getShortInt (self): 
		return self.__processBufferRequest(FormatConstants.SHORT_INT_CODE,
			FormatConstants.SHORT_INT_SIZE)

	def getInt(self):
		return self.__processBufferRequest(FormatConstants.INT_CODE,
			FormatConstants.INT_SIZE)

	def getUnsignedInt(self):
		return self.__processBufferRequest(FormatConstants.UNSIGNED_INT_CODE,
			FormatConstants.UNSIGNED_INT_SIZE)

	def getLongInt(self):
		return self.__processBufferRequest(FormatConstants.LONG_INT_CODE,
			FormatConstants.LONG_INT_SIZE)
	
	def getUnsignedLongInt(self):
		return self.__processBufferRequest(FormatConstants.UNSIGNED_LONG_INT_CODE,
			FormatConstants.UNSIGNED_LONG_INT_SIZE)
	
	def getLongLongInt(self):
		return self.__processBufferRequest(FormatConstants.LONG_LONG_INT_CODE,
			FormatConstants.LONG_LONG_INT_SIZE)
	
	def getUnsignedLongLongInt(self):
		return self.__processBufferRequest(FormatConstants.UNSIGNED_LONG_LONG_INT_CODE,
			FormatConstants.UNSIGNED_LONG_LONG_INT_SIZE)
	
	def getFloat(self):
		return self.__processBufferRequest(FormatConstants.FLOAT_CODE,
			FormatConstants.FLOAT_SIZE)
	
	def getDouble(self):
		return self.__processBufferRequest(FormatConstants.DOUBLE_CODE,
			FormatConstants.DOUBLE_SIZE)
	
	def getString(self):
		lenght = self.__processBufferRequest(FormatConstants.UNSIGNED_INT_CODE,
			FormatConstants.UNSIGNED_INT_SIZE)
		
		value = self.__processBufferRequest(str(lenght-1)+FormatConstants.STRING_CODE, lenght)

		if (lenght != len(value)+1):
			raise BufferHandlerException("String sent is invalid.")
		
		return value

And finally the message.py, used to create messages that wll be sent.

# message.py
from struct import pack_into
from ctypes import create_string_buffer
from constants import FormatConstants

class Message:
	def __init__(self):
		self.formatString = ""
		self.bufferSize = []
		self.valueList = []
		
	def __addTypedValue(self, typeCode, typeSize, value):
		self.formatString += typeCode
		self.bufferSize.append(typeSize)
		self.valueList.append(value)

	def finish(self):
		totalBufferSize = 0
		for i in self.bufferSize:
			totalBufferSize += i
		buff =  create_string_buffer(totalBufferSize)
		offset = 0
		j = 0
		for i in self.valueList:
			if (self.formatString[j] == 's'):
				w = 0
				for k in i:
					pack_into(self.formatString[j], buff, offset+w, k)
					w+=1
			else:
				pack_into(self.formatString[j], buff, offset, i)
			offset += self.bufferSize[j]
			j+=1
		return buff

	def addUnsignedShortInt(self, value):
		self.__addTypedValue(FormatConstants.UNSIGNED_SHORT_INT_CODE, 
			FormatConstants.UNSIGNED_SHORT_INT_SIZE, value)
		
	def addChar(self, value):
		self.__addTypedValue(FormatConstants.CHAR_CODE, 
			FormatConstants.CHAR_SIZE, value)

	def addShortInt (self, value): 
		self.__addTypedValue(FormatConstants.SHORT_INT_CODE, 
			FormatConstants.SHORT_INT_SIZE, value)

	def addInt(self, value):
		self.__addTypedValue(FormatConstants.INT_CODE, 
			FormatConstants.INT_SIZE, value)

	def addUnsignedInt(self, value):
		self.__addTypedValue(FormatConstants.UNSIGNED_INT_CODE, 
			FormatConstants.UNSIGNED_INT_SIZE, value)

	def addLongInt(self, value):
		self.__addTypedValue(FormatConstants.LONG_INT_CODE, 
			FormatConstants.LONG_INT_SIZE, value)
	
	def addUnsignedLongInt(self, value):
		self.__addTypedValue(FormatConstants.UNSIGNED_LONG_INT_CODE, 
			FormatConstants.UNSIGNED_LONG_INT_SIZE, value)
	
	def addLongLongInt(self, value):
		self.__addTypedValue(FormatConstants.LONG_LONG_INT_CODE, 
			FormatConstants.LONG_LONG_INT_SIZE, value)
	
	def addUnsignedLongLongInt(self, value):
		self.__addTypedValue(FormatConstants.UNSIGNED_LONG_LONG_INT_CODE, 
			FormatConstants.UNSIGNED_LONG_LONG_INT_SIZE, value)
	
	def addFloat(self, value):
		self.__addTypedValue(FormatConstants.FLOAT_CODE, 
			FormatConstants.FLOAT_SIZE, value)
	
	def addDouble(self, value):
		self.__addTypedValue(FormatConstants.DOUBLE_CODE, 
			FormatConstants.DOUBLE_SIZE, value)
	
	def addString(self, value):
		self.__addTypedValue(FormatConstants.UNSIGNED_INT_CODE, 
			FormatConstants.UNSIGNED_INT_SIZE, len(value)+1)
		self.__addTypedValue(FormatConstants.STRING_CODE, 
			len(value)+1, value)

I can post a whole example that I plan to release with this code when it is a final version, but I guess it would be far too much code in a single post. So I will use some very simple example.

Assuming we want to send an position update for our character via socket, it would be something like this:

    x = character.getX()
    y = character.getY()
    message = Message()
    message.addShortInt(POSITION_UPDATE_ID)
    message.addFloat(x)
    message.addFloat(y)
    mainSocket.send(message.finish())

Where:

Message is the Message class constructor.

addXXX are methods provided that will add the given variable as the type given. Here is a list of every method provided:

addUnsignedShortInt, addChar, addShortInt, addInt, addUnsignedInt, addLongInt, addUnsignedLongInt, addLongLongInt, addUnsignedLongLongInt, addFloat, addDouble and addString.

finish method creates the buffer via ctypes, fills it as the Message object was configured and returns the buffer.

The shortInt added is used as a message id for a protocol, it is used to tell the client that the message received is a position update and also used for the client to know what is the rest of the message. The other two floats are the (x,y) coords of the character.

To receive the data, the code would be something like this:

    bufferHandler = BufferHandler(4096)
    receivedBytes = receiveSocket.recv_into(bufferHandler.getBuffer(), bufferHandler.getOriginalSize())
    if (receivedBytes != 0):
        bufferHandler.setLimit(receivedBytes)
        messageId = bufferHandler.getShortInt()
        if (messageId == POSITION_UPDATE_ID):
            x = bufferHandler.getFloat()
            y = bufferHandler.getFloat()
            character.setX(x)
            character.setY(y)

Where:

BufferHandler is the class constructor, 4096 is the max length of the the buffer.

getBuffer returs the buffer created in with ctypes module, it will be used to receive the information.

getOriginalSize returs the max size of the buffer.

setLimit is used to parse the received data, if you ask for more information than this set limit, it will raise an exception.

getXXX are methods used to recover information, there is one version for each of the addXXX of the message class.

Well that is it, any feedback or suggestions are very welcome.

As always, you may use this code if you find it useful, but there are no waranties.

Thanks in advance.