back to Reference (Gold) summary
Reference (Gold): bitstring
Pytest Summary for test tests
| status | count |
|---|---|
| passed | 834 |
| skipped | 1 |
| total | 835 |
| collected | 835 |
Failed pytests:
Patch diff
diff --git a/bitstring/array_.py b/bitstring/array_.py
index 5df26f0..5aaed8b 100644
--- a/bitstring/array_.py
+++ b/bitstring/array_.py
@@ -1,4 +1,5 @@
from __future__ import annotations
+
import math
import numbers
from collections.abc import Sized
@@ -14,7 +15,10 @@ import array
import operator
import io
import sys
+
+# The possible types stored in each element of the Array
ElementType = Union[float, str, int, bytes, bool, Bits]
+
options = Options()
@@ -63,24 +67,19 @@ class Array:
"""
- def __init__(self, dtype: Union[str, Dtype], initializer: Optional[
- Union[int, Array, array.array, Iterable, Bits, bytes, bytearray,
- memoryview, BinaryIO]]=None, trailing_bits: Optional[BitsType]=None
- ) ->None:
+ def __init__(self, dtype: Union[str, Dtype], initializer: Optional[Union[int, Array, array.array, Iterable, Bits, bytes, bytearray, memoryview, BinaryIO]] = None,
+ trailing_bits: Optional[BitsType] = None) -> None:
self.data = BitArray()
if isinstance(dtype, Dtype) and dtype.scale == 'auto':
- if isinstance(initializer, (int, Bits, bytes, bytearray,
- memoryview, BinaryIO)):
- raise TypeError(
- "An Array with an 'auto' scale factor can only be created from an iterable of values."
- )
- auto_scale = self._calculate_auto_scale(initializer, dtype.name,
- dtype.length)
+ if isinstance(initializer, (int, Bits, bytes, bytearray, memoryview, BinaryIO)):
+ raise TypeError("An Array with an 'auto' scale factor can only be created from an iterable of values.")
+ auto_scale = self._calculate_auto_scale(initializer, dtype.name, dtype.length)
dtype = Dtype(dtype.name, dtype.length, scale=auto_scale)
try:
self._set_dtype(dtype)
except ValueError as e:
raise CreationError(e)
+
if isinstance(initializer, numbers.Integral):
self.data = BitArray(initializer * self._dtype.bitlength)
elif isinstance(initializer, (Bits, bytes, bytearray, memoryview)):
@@ -89,148 +88,256 @@ class Array:
self.fromfile(initializer)
elif initializer is not None:
self.extend(initializer)
+
if trailing_bits is not None:
self.data += BitArray._create_from_bitstype(trailing_bits)
+
_largest_values = None
- def _create_element(self, value: ElementType) ->Bits:
+ @staticmethod
+ def _calculate_auto_scale(initializer, name: str, length: Optional[int]) -> float:
+ # Now need to find the largest power of 2 representable with this format.
+ if Array._largest_values is None:
+ Array._largest_values = {
+ 'mxint8': Bits('0b01111111').mxint8, # 1.0 + 63.0/64.0,
+ 'e2m1mxfp4': Bits('0b0111').e2m1mxfp4, # 6.0
+ 'e2m3mxfp6': Bits('0b011111').e2m3mxfp6, # 7.5
+ 'e3m2mxfp6': Bits('0b011111').e3m2mxfp6, # 28.0
+ 'e4m3mxfp8': Bits('0b01111110').e4m3mxfp8, # 448.0
+ 'e5m2mxfp8': Bits('0b01111011').e5m2mxfp8, # 57344.0
+ 'p4binary8': Bits('0b01111110').p4binary8, # 224.0
+ 'p3binary8': Bits('0b01111110').p3binary8, # 49152.0
+ 'float16': Bits('0x7bff').float16, # 65504.0
+ # The bfloat range is so large the scaling algorithm doesn't work well, so I'm disallowing it.
+ # 'bfloat16': Bits('0x7f7f').bfloat16, # 3.38953139e38,
+ }
+ if f'{name}{length}' in Array._largest_values.keys():
+ float_values = Array('float64', initializer).tolist()
+ if not float_values:
+ raise ValueError("Can't calculate an 'auto' scale with an empty Array initializer.")
+ max_float_value = max(abs(x) for x in float_values)
+ if max_float_value == 0:
+ # This special case isn't covered in the standard. I'm choosing to return no scale.
+ return 1.0
+ # We need to find the largest power of 2 that is less than the max value
+ log2 = math.floor(math.log2(max_float_value))
+ lp2 = math.floor(math.log2(Array._largest_values[f'{name}{length}']))
+ lg_scale = log2 - lp2
+ # Saturate at values representable in E8M0 format.
+ if lg_scale > 127:
+ lg_scale = 127
+ elif lg_scale < -127:
+ lg_scale = -127
+ return 2 ** lg_scale
+ else:
+ raise ValueError(f"Can't calculate auto scale for format '{name}{length}'. "
+ f"This feature is only available for these formats: {list(Array._largest_values.keys())}.")
+
+ @property
+ def itemsize(self) -> int:
+ return self._dtype.length
+
+ @property
+ def trailing_bits(self) -> BitArray:
+ trailing_bit_length = len(self.data) % self._dtype.bitlength
+ return BitArray() if trailing_bit_length == 0 else self.data[-trailing_bit_length:]
+
+ @property
+ def dtype(self) -> Dtype:
+ return self._dtype
+
+ @dtype.setter
+ def dtype(self, new_dtype: Union[str, Dtype]) -> None:
+ self._set_dtype(new_dtype)
+
+ def _set_dtype(self, new_dtype: Union[str, Dtype]) -> None:
+ if isinstance(new_dtype, Dtype):
+ self._dtype = new_dtype
+ else:
+ try:
+ dtype = Dtype(new_dtype)
+ except ValueError:
+ name_length = utils.parse_single_struct_token(new_dtype)
+ if name_length is not None:
+ dtype = Dtype(name_length[0], name_length[1])
+ else:
+ raise ValueError(f"Inappropriate Dtype for Array: '{new_dtype}'.")
+ if dtype.length is None:
+ raise ValueError(f"A fixed length format is needed for an Array, received '{new_dtype}'.")
+ self._dtype = dtype
+ if self._dtype.scale == 'auto':
+ raise ValueError("A Dtype with an 'auto' scale factor can only be used when creating a new Array.")
+
+ def _create_element(self, value: ElementType) -> Bits:
"""Create Bits from value according to the token_name and token_length"""
- pass
+ b = self._dtype.build(value)
+ if len(b) != self._dtype.length:
+ raise ValueError(f"The value {value!r} has the wrong length for the format '{self._dtype}'.")
+ return b
- def __len__(self) ->int:
+ def __len__(self) -> int:
return len(self.data) // self._dtype.length
@overload
- def __getitem__(self, key: slice) ->Array:
+ def __getitem__(self, key: slice) -> Array:
...
@overload
- def __getitem__(self, key: int) ->ElementType:
+ def __getitem__(self, key: int) -> ElementType:
...
- def __getitem__(self, key: Union[slice, int]) ->Union[Array, ElementType]:
+ def __getitem__(self, key: Union[slice, int]) -> Union[Array, ElementType]:
if isinstance(key, slice):
start, stop, step = key.indices(len(self))
if step != 1:
d = BitArray()
- for s in range(start * self._dtype.length, stop * self.
- _dtype.length, step * self._dtype.length):
- d.append(self.data[s:s + self._dtype.length])
+ for s in range(start * self._dtype.length, stop * self._dtype.length, step * self._dtype.length):
+ d.append(self.data[s: s + self._dtype.length])
a = self.__class__(self._dtype)
a.data = d
return a
else:
a = self.__class__(self._dtype)
- a.data = self.data[start * self._dtype.length:stop * self.
- _dtype.length]
+ a.data = self.data[start * self._dtype.length: stop * self._dtype.length]
return a
else:
if key < 0:
key += len(self)
if key < 0 or key >= len(self):
- raise IndexError(
- f'Index {key} out of range for Array of length {len(self)}.'
- )
- return self._dtype.read_fn(self.data, start=self._dtype.length *
- key)
+ raise IndexError(f"Index {key} out of range for Array of length {len(self)}.")
+ return self._dtype.read_fn(self.data, start=self._dtype.length * key)
@overload
- def __setitem__(self, key: slice, value: Iterable[ElementType]) ->None:
+ def __setitem__(self, key: slice, value: Iterable[ElementType]) -> None:
...
@overload
- def __setitem__(self, key: int, value: ElementType) ->None:
+ def __setitem__(self, key: int, value: ElementType) -> None:
...
- def __setitem__(self, key: Union[slice, int], value: Union[Iterable[
- ElementType], ElementType]) ->None:
+ def __setitem__(self, key: Union[slice, int], value: Union[Iterable[ElementType], ElementType]) -> None:
if isinstance(key, slice):
start, stop, step = key.indices(len(self))
if not isinstance(value, Iterable):
- raise TypeError('Can only assign an iterable to a slice.')
+ raise TypeError("Can only assign an iterable to a slice.")
if step == 1:
new_data = BitArray()
for x in value:
new_data += self._create_element(x)
- self.data[start * self._dtype.length:stop * self._dtype.length
- ] = new_data
+ self.data[start * self._dtype.length: stop * self._dtype.length] = new_data
return
items_in_slice = len(range(start, stop, step))
if not isinstance(value, Sized):
value = list(value)
if len(value) == items_in_slice:
for s, v in zip(range(start, stop, step), value):
- self.data.overwrite(self._create_element(v), s * self.
- _dtype.length)
+ self.data.overwrite(self._create_element(v), s * self._dtype.length)
else:
- raise ValueError(
- f"Can't assign {len(value)} values to an extended slice of length {items_in_slice}."
- )
+ raise ValueError(f"Can't assign {len(value)} values to an extended slice of length {items_in_slice}.")
else:
if key < 0:
key += len(self)
if key < 0 or key >= len(self):
- raise IndexError(
- f'Index {key} out of range for Array of length {len(self)}.'
- )
+ raise IndexError(f"Index {key} out of range for Array of length {len(self)}.")
start = self._dtype.length * key
self.data.overwrite(self._create_element(value), start)
return
- def __delitem__(self, key: Union[slice, int]) ->None:
+ def __delitem__(self, key: Union[slice, int]) -> None:
if isinstance(key, slice):
start, stop, step = key.indices(len(self))
if step == 1:
- self.data.__delitem__(slice(start * self._dtype.length,
- stop * self._dtype.length))
+ self.data.__delitem__(slice(start * self._dtype.length, stop * self._dtype.length))
return
- r = reversed(range(start, stop, step)) if step > 0 else range(start
- , stop, step)
+ # We need to delete from the end or the earlier positions will change
+ r = reversed(range(start, stop, step)) if step > 0 else range(start, stop, step)
for s in r:
- self.data.__delitem__(slice(s * self._dtype.length, (s + 1) *
- self._dtype.length))
+ self.data.__delitem__(slice(s * self._dtype.length, (s + 1) * self._dtype.length))
else:
if key < 0:
key += len(self)
if key < 0 or key >= len(self):
raise IndexError
start = self._dtype.length * key
- del self.data[start:start + self._dtype.length]
+ del self.data[start: start + self._dtype.length]
- def __repr__(self) ->str:
- list_str = f'{self.tolist()}'
+ def __repr__(self) -> str:
+ list_str = f"{self.tolist()}"
trailing_bit_length = len(self.data) % self._dtype.length
- final_str = ('' if trailing_bit_length == 0 else ', trailing_bits=' +
- repr(self.data[-trailing_bit_length:]))
+ final_str = "" if trailing_bit_length == 0 else ", trailing_bits=" + repr(
+ self.data[-trailing_bit_length:])
return f"Array('{self._dtype}', {list_str}{final_str})"
- def astype(self, dtype: Union[str, Dtype]) ->Array:
+ def astype(self, dtype: Union[str, Dtype]) -> Array:
"""Return Array with elements of new dtype, initialised from current Array."""
- pass
+ new_array = self.__class__(dtype, self.tolist())
+ return new_array
+
+ def tolist(self) -> List[ElementType]:
+ return [self._dtype.read_fn(self.data, start=start)
+ for start in range(0, len(self.data) - self._dtype.length + 1, self._dtype.length)]
+
+ def append(self, x: ElementType) -> None:
+ if len(self.data) % self._dtype.length != 0:
+ raise ValueError("Cannot append to Array as its length is not a multiple of the format length.")
+ self.data += self._create_element(x)
+
+ def extend(self, iterable: Union[Array, array.array, Iterable[Any]]) -> None:
+ if len(self.data) % self._dtype.length != 0:
+ raise ValueError(f"Cannot extend Array as its data length ({len(self.data)} bits) is not a multiple of the format length ({self._dtype.length} bits).")
+ if isinstance(iterable, Array):
+ if self._dtype.name != iterable._dtype.name or self._dtype.length != iterable._dtype.length:
+ raise TypeError(
+ f"Cannot extend an Array with format '{self._dtype}' from an Array of format '{iterable._dtype}'.")
+ # No need to iterate over the elements, we can just append the data
+ self.data.append(iterable.data)
+ elif isinstance(iterable, array.array):
+ # array.array types are always native-endian, hence the '='
+ name_value = utils.parse_single_struct_token('=' + iterable.typecode)
+ if name_value is None:
+ raise ValueError(f"Cannot extend from array with typecode {iterable.typecode}.")
+ other_dtype = dtype_register.get_dtype(*name_value, scale=None)
+ if self._dtype.name != other_dtype.name or self._dtype.length != other_dtype.length:
+ raise ValueError(
+ f"Cannot extend an Array with format '{self._dtype}' from an array with typecode '{iterable.typecode}'.")
+ self.data += iterable.tobytes()
+ else:
+ if isinstance(iterable, str):
+ raise TypeError("Can't extend an Array with a str.")
+ for item in iterable:
+ self.data += self._create_element(item)
- def insert(self, i: int, x: ElementType) ->None:
+ def insert(self, i: int, x: ElementType) -> None:
"""Insert a new element into the Array at position i.
"""
- pass
+ i = min(i, len(self)) # Inserting beyond len of array inserts at the end (copying standard behaviour)
+ self.data.insert(self._create_element(x), i * self._dtype.length)
- def pop(self, i: int=-1) ->ElementType:
+ def pop(self, i: int = -1) -> ElementType:
"""Return and remove an element of the Array.
Default is to return and remove the final element.
"""
- pass
+ if len(self) == 0:
+ raise IndexError("Can't pop from an empty Array.")
+ x = self[i]
+ del self[i]
+ return x
- def byteswap(self) ->None:
+ def byteswap(self) -> None:
"""Change the endianness in-place of all items in the Array.
If the Array format is not a whole number of bytes a ValueError will be raised.
"""
- pass
+ if self._dtype.length % 8 != 0:
+ raise ValueError(
+ f"byteswap can only be used for whole-byte elements. The '{self._dtype}' format is {self._dtype.length} bits long.")
+ self.data.byteswap(self.itemsize // 8)
- def count(self, value: ElementType) ->int:
+ def count(self, value: ElementType) -> int:
"""Return count of Array items that equal value.
value -- The quantity to compare each Array element to. Type should be appropriate for the Array format.
@@ -238,26 +345,52 @@ class Array:
For floating point types using a value of float('nan') will count the number of elements that are NaN.
"""
- pass
+ if math.isnan(value):
+ return sum(math.isnan(i) for i in self)
+ else:
+ return sum(i == value for i in self)
- def tobytes(self) ->bytes:
+ def tobytes(self) -> bytes:
"""Return the Array data as a bytes object, padding with zero bits if needed.
Up to seven zero bits will be added at the end to byte align.
"""
- pass
+ return self.data.tobytes()
- def tofile(self, f: BinaryIO) ->None:
+ def tofile(self, f: BinaryIO) -> None:
"""Write the Array data to a file object, padding with zero bits if needed.
Up to seven zero bits will be added at the end to byte align.
"""
- pass
+ self.data.tofile(f)
+
+ def fromfile(self, f: BinaryIO, n: Optional[int] = None) -> None:
+ trailing_bit_length = len(self.data) % self._dtype.bitlength
+ if trailing_bit_length != 0:
+ raise ValueError(f"Cannot extend Array as its data length ({len(self.data)} bits) is not a multiple of the format length ({self._dtype.bitlength} bits).")
- def pp(self, fmt: Optional[str]=None, width: int=120, show_offset: bool
- =True, stream: TextIO=sys.stdout) ->None:
+ new_data = Bits(f)
+ max_items = len(new_data) // self._dtype.length
+ items_to_append = max_items if n is None else min(n, max_items)
+ self.data += new_data[0: items_to_append * self._dtype.bitlength]
+ if n is not None and items_to_append < n:
+ raise EOFError(f"Only {items_to_append} were appended, not the {n} items requested.")
+
+ def reverse(self) -> None:
+ trailing_bit_length = len(self.data) % self._dtype.length
+ if trailing_bit_length != 0:
+ raise ValueError(f"Cannot reverse the items in the Array as its data length ({len(self.data)} bits) is not a multiple of the format length ({self._dtype.length} bits).")
+ for start_bit in range(0, len(self.data) // 2, self._dtype.length):
+ start_swap_bit = len(self.data) - start_bit - self._dtype.length
+ temp = self.data[start_bit: start_bit + self._dtype.length]
+ self.data[start_bit: start_bit + self._dtype.length] = self.data[
+ start_swap_bit: start_swap_bit + self._dtype.length]
+ self.data[start_swap_bit: start_swap_bit + self._dtype.length] = temp
+
+ def pp(self, fmt: Optional[str] = None, width: int = 120,
+ show_offset: bool = True, stream: TextIO = sys.stdout) -> None:
"""Pretty-print the Array contents.
fmt -- Data format string. Defaults to current Array dtype.
@@ -267,44 +400,183 @@ class Array:
stream -- A TextIO object with a write() method. Defaults to sys.stdout.
"""
- pass
-
- def equals(self, other: Any) ->bool:
+ colour = Colour(not options.no_color)
+ sep = ' '
+ dtype2 = None
+ tidy_fmt = None
+ if fmt is None:
+ fmt = self.dtype
+ dtype1 = self.dtype
+ tidy_fmt = "dtype='" + colour.purple + str(self.dtype) + "'" + colour.off
+ else:
+ token_list = utils.preprocess_tokens(fmt)
+ if len(token_list) not in [1, 2]:
+ raise ValueError(f"Only one or two tokens can be used in an Array.pp() format - '{fmt}' has {len(token_list)} tokens.")
+ name1, length1 = utils.parse_name_length_token(token_list[0])
+ dtype1 = Dtype(name1, length1)
+ if len(token_list) == 2:
+ name2, length2 = utils.parse_name_length_token(token_list[1])
+ dtype2 = Dtype(name2, length2)
+
+ token_length = dtype1.bitlength
+ if dtype2 is not None:
+ # For two types we're OK as long as they don't have different lengths given.
+ if dtype1.bitlength is not None and dtype2.bitlength is not None and dtype1.bitlength != dtype2.bitlength:
+ raise ValueError(f"Two different format lengths specified ('{fmt}'). Either specify just one, or two the same length.")
+ if token_length is None:
+ token_length = dtype2.bitlength
+ if token_length is None:
+ token_length = self.itemsize
+
+ trailing_bit_length = len(self.data) % token_length
+ format_sep = " : " # String to insert on each line between multiple formats
+ if tidy_fmt is None:
+ tidy_fmt = colour.purple + str(dtype1) + colour.off
+ if dtype2 is not None:
+ tidy_fmt += ', ' + colour.blue + str(dtype2) + colour.off
+ tidy_fmt = "fmt='" + tidy_fmt + "'"
+ data = self.data if trailing_bit_length == 0 else self.data[0: -trailing_bit_length]
+ length = len(self.data) // token_length
+ len_str = colour.green + str(length) + colour.off
+ stream.write(f"<{self.__class__.__name__} {tidy_fmt}, length={len_str}, itemsize={token_length} bits, total data size={(len(self.data) + 7) // 8} bytes> [\n")
+ data._pp(dtype1, dtype2, token_length, width, sep, format_sep, show_offset, stream, False, token_length)
+ stream.write("]")
+ if trailing_bit_length != 0:
+ stream.write(" + trailing_bits = " + str(self.data[-trailing_bit_length:]))
+ stream.write("\n")
+
+ def equals(self, other: Any) -> bool:
"""Return True if format and all Array items are equal."""
- pass
-
- def __iter__(self) ->Iterable[ElementType]:
+ if isinstance(other, Array):
+ if self._dtype.length != other._dtype.length:
+ return False
+ if self._dtype.name != other._dtype.name:
+ return False
+ if self.data != other.data:
+ return False
+ return True
+ elif isinstance(other, array.array):
+ # Assume we are comparing with an array type
+ if self.trailing_bits:
+ return False
+ # array's itemsize is in bytes, not bits.
+ if self.itemsize != other.itemsize * 8:
+ return False
+ if len(self) != len(other):
+ return False
+ if self.tolist() != other.tolist():
+ return False
+ return True
+ return False
+
+ def __iter__(self) -> Iterable[ElementType]:
start = 0
for _ in range(len(self)):
yield self._dtype.read_fn(self.data, start=start)
start += self._dtype.length
- def __copy__(self) ->Array:
+ def __copy__(self) -> Array:
a_copy = self.__class__(self._dtype)
a_copy.data = copy.copy(self.data)
return a_copy
- def _apply_op_to_all_elements(self, op, value: Union[int, float, None],
- is_comparison: bool=False) ->Array:
+ def _apply_op_to_all_elements(self, op, value: Union[int, float, None], is_comparison: bool = False) -> Array:
"""Apply op with value to each element of the Array and return a new Array"""
- pass
-
- def _apply_op_to_all_elements_inplace(self, op, value: Union[int, float]
- ) ->Array:
+ new_array = self.__class__('bool' if is_comparison else self._dtype)
+ new_data = BitArray()
+ failures = index = 0
+ msg = ''
+ if value is not None:
+ def partial_op(a):
+ return op(a, value)
+ else:
+ def partial_op(a):
+ return op(a)
+ for i in range(len(self)):
+ v = self._dtype.read_fn(self.data, start=self._dtype.length * i)
+ try:
+ new_data.append(new_array._create_element(partial_op(v)))
+ except (CreationError, ZeroDivisionError, ValueError) as e:
+ if failures == 0:
+ msg = str(e)
+ index = i
+ failures += 1
+ if failures != 0:
+ raise ValueError(f"Applying operator '{op.__name__}' to Array caused {failures} errors. "
+ f'First error at index {index} was: "{msg}"')
+ new_array.data = new_data
+ return new_array
+
+ def _apply_op_to_all_elements_inplace(self, op, value: Union[int, float]) -> Array:
"""Apply op with value to each element of the Array in place."""
- pass
-
- def _apply_bitwise_op_to_all_elements(self, op, value: BitsType) ->Array:
+ # This isn't really being done in-place, but it's simpler and faster for now?
+ new_data = BitArray()
+ failures = index = 0
+ msg = ''
+ for i in range(len(self)):
+ v = self._dtype.read_fn(self.data, start=self._dtype.length * i)
+ try:
+ new_data.append(self._create_element(op(v, value)))
+ except (CreationError, ZeroDivisionError, ValueError) as e:
+ if failures == 0:
+ msg = str(e)
+ index = i
+ failures += 1
+ if failures != 0:
+ raise ValueError(f"Applying operator '{op.__name__}' to Array caused {failures} errors. "
+ f'First error at index {index} was: "{msg}"')
+ self.data = new_data
+ return self
+
+ def _apply_bitwise_op_to_all_elements(self, op, value: BitsType) -> Array:
"""Apply op with value to each element of the Array as an unsigned integer and return a new Array"""
- pass
+ a_copy = self[:]
+ a_copy._apply_bitwise_op_to_all_elements_inplace(op, value)
+ return a_copy
- def _apply_bitwise_op_to_all_elements_inplace(self, op, value: BitsType
- ) ->Array:
+ def _apply_bitwise_op_to_all_elements_inplace(self, op, value: BitsType) -> Array:
"""Apply op with value to each element of the Array as an unsigned integer in place."""
- pass
+ value = BitArray._create_from_bitstype(value)
+ if len(value) != self._dtype.length:
+ raise ValueError(f"Bitwise op needs a bitstring of length {self._dtype.length} to match format {self._dtype}.")
+ for start in range(0, len(self) * self._dtype.length, self._dtype.length):
+ self.data[start: start + self._dtype.length] = op(self.data[start: start + self._dtype.length], value)
+ return self
+
+ def _apply_op_between_arrays(self, op, other: Array, is_comparison: bool = False) -> Array:
+ if len(self) != len(other):
+ msg = f"Cannot operate element-wise on Arrays with different lengths ({len(self)} and {len(other)})."
+ if op in [operator.add, operator.iadd]:
+ msg += " Use extend() method to concatenate Arrays."
+ if op in [operator.eq, operator.ne]:
+ msg += " Use equals() method to compare Arrays for a single boolean result."
+ raise ValueError(msg)
+ if is_comparison:
+ new_type = dtype_register.get_dtype('bool', 1)
+ else:
+ new_type = self._promotetype(self._dtype, other._dtype)
+ new_array = self.__class__(new_type)
+ new_data = BitArray()
+ failures = index = 0
+ msg = ''
+ for i in range(len(self)):
+ a = self._dtype.read_fn(self.data, start=self._dtype.length * i)
+ b = other._dtype.read_fn(other.data, start=other._dtype.length * i)
+ try:
+ new_data.append(new_array._create_element(op(a, b)))
+ except (CreationError, ValueError, ZeroDivisionError) as e:
+ if failures == 0:
+ msg = str(e)
+ index = i
+ failures += 1
+ if failures != 0:
+ raise ValueError(f"Applying operator '{op.__name__}' between Arrays caused {failures} errors. "
+ f'First error at index {index} was: "{msg}"')
+ new_array.data = new_data
+ return new_array
@classmethod
- def _promotetype(cls, type1: Dtype, type2: Dtype) ->Dtype:
+ def _promotetype(cls, type1: Dtype, type2: Dtype) -> Dtype:
"""When combining types which one wins?
1. We only deal with types representing floats or integers.
@@ -315,165 +587,196 @@ class Array:
6. In a tie the first type wins against the second type.
"""
- pass
-
- def __add__(self, other: Union[int, float, Array]) ->Array:
+ def is_float(x): return x.return_type is float
+ def is_int(x): return x.return_type is int or x.return_type is bool
+ if is_float(type1) + is_int(type1) + is_float(type2) + is_int(type2) != 2:
+ raise ValueError(f"Only integer and floating point types can be combined - not '{type1}' and '{type2}'.")
+ # If same type choose the widest
+ if type1.name == type2.name:
+ return type1 if type1.length > type2.length else type2
+ # We choose floats above integers, irrespective of the widths
+ if is_float(type1) and is_int(type2):
+ return type1
+ if is_int(type1) and is_float(type2):
+ return type2
+ if is_float(type1) and is_float(type2):
+ return type2 if type2.length > type1.length else type1
+ assert is_int(type1) and is_int(type2)
+ if type1.is_signed and not type2.is_signed:
+ return type1
+ if type2.is_signed and not type1.is_signed:
+ return type2
+ return type2 if type2.length > type1.length else type1
+
+ # Operators between Arrays or an Array and scalar value
+
+ def __add__(self, other: Union[int, float, Array]) -> Array:
"""Add int or float to all elements."""
if isinstance(other, Array):
return self._apply_op_between_arrays(operator.add, other)
return self._apply_op_to_all_elements(operator.add, other)
- def __iadd__(self, other: Union[int, float, Array]) ->Array:
+ def __iadd__(self, other: Union[int, float, Array]) -> Array:
if isinstance(other, Array):
return self._apply_op_between_arrays(operator.add, other)
return self._apply_op_to_all_elements_inplace(operator.add, other)
- def __isub__(self, other: Union[int, float, Array]) ->Array:
+ def __isub__(self, other: Union[int, float, Array]) -> Array:
if isinstance(other, Array):
return self._apply_op_between_arrays(operator.sub, other)
return self._apply_op_to_all_elements_inplace(operator.sub, other)
- def __sub__(self, other: Union[int, float, Array]) ->Array:
+ def __sub__(self, other: Union[int, float, Array]) -> Array:
if isinstance(other, Array):
return self._apply_op_between_arrays(operator.sub, other)
return self._apply_op_to_all_elements(operator.sub, other)
- def __mul__(self, other: Union[int, float, Array]) ->Array:
+ def __mul__(self, other: Union[int, float, Array]) -> Array:
if isinstance(other, Array):
return self._apply_op_between_arrays(operator.mul, other)
return self._apply_op_to_all_elements(operator.mul, other)
- def __imul__(self, other: Union[int, float, Array]) ->Array:
+ def __imul__(self, other: Union[int, float, Array]) -> Array:
if isinstance(other, Array):
return self._apply_op_between_arrays(operator.mul, other)
return self._apply_op_to_all_elements_inplace(operator.mul, other)
- def __floordiv__(self, other: Union[int, float, Array]) ->Array:
+ def __floordiv__(self, other: Union[int, float, Array]) -> Array:
if isinstance(other, Array):
return self._apply_op_between_arrays(operator.floordiv, other)
return self._apply_op_to_all_elements(operator.floordiv, other)
- def __ifloordiv__(self, other: Union[int, float, Array]) ->Array:
+ def __ifloordiv__(self, other: Union[int, float, Array]) -> Array:
if isinstance(other, Array):
return self._apply_op_between_arrays(operator.floordiv, other)
return self._apply_op_to_all_elements_inplace(operator.floordiv, other)
- def __truediv__(self, other: Union[int, float, Array]) ->Array:
+ def __truediv__(self, other: Union[int, float, Array]) -> Array:
if isinstance(other, Array):
return self._apply_op_between_arrays(operator.truediv, other)
return self._apply_op_to_all_elements(operator.truediv, other)
- def __itruediv__(self, other: Union[int, float, Array]) ->Array:
+ def __itruediv__(self, other: Union[int, float, Array]) -> Array:
if isinstance(other, Array):
return self._apply_op_between_arrays(operator.truediv, other)
return self._apply_op_to_all_elements_inplace(operator.truediv, other)
- def __rshift__(self, other: Union[int, Array]) ->Array:
+ def __rshift__(self, other: Union[int, Array]) -> Array:
if isinstance(other, Array):
return self._apply_op_between_arrays(operator.rshift, other)
return self._apply_op_to_all_elements(operator.rshift, other)
- def __lshift__(self, other: Union[int, Array]) ->Array:
+ def __lshift__(self, other: Union[int, Array]) -> Array:
if isinstance(other, Array):
return self._apply_op_between_arrays(operator.lshift, other)
return self._apply_op_to_all_elements(operator.lshift, other)
- def __irshift__(self, other: Union[int, Array]) ->Array:
+ def __irshift__(self, other: Union[int, Array]) -> Array:
if isinstance(other, Array):
return self._apply_op_between_arrays(operator.rshift, other)
return self._apply_op_to_all_elements_inplace(operator.rshift, other)
- def __ilshift__(self, other: Union[int, Array]) ->Array:
+ def __ilshift__(self, other: Union[int, Array]) -> Array:
if isinstance(other, Array):
return self._apply_op_between_arrays(operator.lshift, other)
return self._apply_op_to_all_elements_inplace(operator.lshift, other)
- def __mod__(self, other: Union[int, Array]) ->Array:
+ def __mod__(self, other: Union[int, Array]) -> Array:
if isinstance(other, Array):
return self._apply_op_between_arrays(operator.mod, other)
return self._apply_op_to_all_elements(operator.mod, other)
- def __imod__(self, other: Union[int, Array]) ->Array:
+ def __imod__(self, other: Union[int, Array]) -> Array:
if isinstance(other, Array):
return self._apply_op_between_arrays(operator.mod, other)
return self._apply_op_to_all_elements_inplace(operator.mod, other)
- def __and__(self, other: BitsType) ->Array:
+ # Bitwise operators
+
+ def __and__(self, other: BitsType) -> Array:
return self._apply_bitwise_op_to_all_elements(operator.iand, other)
- def __iand__(self, other: BitsType) ->Array:
- return self._apply_bitwise_op_to_all_elements_inplace(operator.iand,
- other)
+ def __iand__(self, other: BitsType) -> Array:
+ return self._apply_bitwise_op_to_all_elements_inplace(operator.iand, other)
- def __or__(self, other: BitsType) ->Array:
+ def __or__(self, other: BitsType) -> Array:
return self._apply_bitwise_op_to_all_elements(operator.ior, other)
- def __ior__(self, other: BitsType) ->Array:
- return self._apply_bitwise_op_to_all_elements_inplace(operator.ior,
- other)
+ def __ior__(self, other: BitsType) -> Array:
+ return self._apply_bitwise_op_to_all_elements_inplace(operator.ior, other)
- def __xor__(self, other: BitsType) ->Array:
+ def __xor__(self, other: BitsType) -> Array:
return self._apply_bitwise_op_to_all_elements(operator.ixor, other)
- def __ixor__(self, other: BitsType) ->Array:
- return self._apply_bitwise_op_to_all_elements_inplace(operator.ixor,
- other)
+ def __ixor__(self, other: BitsType) -> Array:
+ return self._apply_bitwise_op_to_all_elements_inplace(operator.ixor, other)
- def __rmul__(self, other: Union[int, float]) ->Array:
+ # Reverse operators between a scalar value and an Array
+
+ def __rmul__(self, other: Union[int, float]) -> Array:
return self._apply_op_to_all_elements(operator.mul, other)
- def __radd__(self, other: Union[int, float]) ->Array:
+ def __radd__(self, other: Union[int, float]) -> Array:
return self._apply_op_to_all_elements(operator.add, other)
- def __rsub__(self, other: Union[int, float]) ->Array:
+ def __rsub__(self, other: Union[int, float]) -> Array:
+ # i - A == (-A) + i
neg = self._apply_op_to_all_elements(operator.neg, None)
return neg._apply_op_to_all_elements(operator.add, other)
- def __rand__(self, other: BitsType) ->Array:
+ # Reverse operators between a scalar and something that can be a BitArray.
+
+ def __rand__(self, other: BitsType) -> Array:
return self._apply_bitwise_op_to_all_elements(operator.iand, other)
- def __ror__(self, other: BitsType) ->Array:
+ def __ror__(self, other: BitsType) -> Array:
return self._apply_bitwise_op_to_all_elements(operator.ior, other)
- def __rxor__(self, other: BitsType) ->Array:
+ def __rxor__(self, other: BitsType) -> Array:
return self._apply_bitwise_op_to_all_elements(operator.ixor, other)
- def __lt__(self, other: Union[int, float, Array]) ->Array:
+ # Comparison operators
+
+ def __lt__(self, other: Union[int, float, Array]) -> Array:
if isinstance(other, Array):
- return self._apply_op_between_arrays(operator.lt, other,
- is_comparison=True)
- return self._apply_op_to_all_elements(operator.lt, other,
- is_comparison=True)
+ return self._apply_op_between_arrays(operator.lt, other, is_comparison=True)
+ return self._apply_op_to_all_elements(operator.lt, other, is_comparison=True)
- def __gt__(self, other: Union[int, float, Array]) ->Array:
+ def __gt__(self, other: Union[int, float, Array]) -> Array:
if isinstance(other, Array):
- return self._apply_op_between_arrays(operator.gt, other,
- is_comparison=True)
- return self._apply_op_to_all_elements(operator.gt, other,
- is_comparison=True)
+ return self._apply_op_between_arrays(operator.gt, other, is_comparison=True)
+ return self._apply_op_to_all_elements(operator.gt, other, is_comparison=True)
- def __ge__(self, other: Union[int, float, Array]) ->Array:
+ def __ge__(self, other: Union[int, float, Array]) -> Array:
if isinstance(other, Array):
- return self._apply_op_between_arrays(operator.ge, other,
- is_comparison=True)
- return self._apply_op_to_all_elements(operator.ge, other,
- is_comparison=True)
+ return self._apply_op_between_arrays(operator.ge, other, is_comparison=True)
+ return self._apply_op_to_all_elements(operator.ge, other, is_comparison=True)
- def __le__(self, other: Union[int, float, Array]) ->Array:
+ def __le__(self, other: Union[int, float, Array]) -> Array:
if isinstance(other, Array):
- return self._apply_op_between_arrays(operator.le, other,
- is_comparison=True)
- return self._apply_op_to_all_elements(operator.le, other,
- is_comparison=True)
+ return self._apply_op_between_arrays(operator.le, other, is_comparison=True)
+ return self._apply_op_to_all_elements(operator.le, other, is_comparison=True)
+
+ def _eq_ne(self, op, other: Any) -> Array:
+ if isinstance(other, (int, float, str, Bits)):
+ return self._apply_op_to_all_elements(op, other, is_comparison=True)
+ try:
+ other = self.__class__(self.dtype, other)
+ except:
+ return NotImplemented
+ finally:
+ return self._apply_op_between_arrays(op, other, is_comparison=True)
- def __eq__(self, other: Any) ->Array:
+ def __eq__(self, other: Any) -> Array:
return self._eq_ne(operator.eq, other)
- def __ne__(self, other: Any) ->Array:
+ def __ne__(self, other: Any) -> Array:
return self._eq_ne(operator.ne, other)
+ # Unary operators
+
def __neg__(self):
return self._apply_op_to_all_elements(operator.neg, None)
def __abs__(self):
- return self._apply_op_to_all_elements(operator.abs, None)
+ return self._apply_op_to_all_elements(operator.abs, None)
\ No newline at end of file
diff --git a/bitstring/bitarray_.py b/bitstring/bitarray_.py
index 5d6204f..4e68684 100644
--- a/bitstring/bitarray_.py
+++ b/bitstring/bitarray_.py
@@ -1,4 +1,5 @@
from __future__ import annotations
+
import copy
import numbers
import re
@@ -7,6 +8,7 @@ from typing import Union, List, Iterable, Any, Optional
from bitstring import utils
from bitstring.exceptions import CreationError, Error
from bitstring.bits import Bits, BitsType, TBits
+
import bitstring.dtypes
@@ -64,11 +66,14 @@ class BitArray(Bits):
len -- Length of the bitstring in bits.
"""
+
__slots__ = ()
+
+ # As BitArray objects are mutable, we shouldn't allow them to be hashed.
__hash__: None = None
- def __init__(self, auto: Optional[Union[BitsType, int]]=None, /, length:
- Optional[int]=None, offset: Optional[int]=None, **kwargs) ->None:
+ def __init__(self, auto: Optional[Union[BitsType, int]] = None, /, length: Optional[int] = None,
+ offset: Optional[int] = None, **kwargs) -> None:
"""Either specify an 'auto' initialiser:
A string of comma separated tokens, an integer, a file object,
a bytearray, a boolean iterable or another bitstring.
@@ -111,29 +116,27 @@ class BitArray(Bits):
self._bitstore = self._bitstore._copy()
self._bitstore.immutable = False
- def copy(self: TBits) ->TBits:
+ def copy(self: TBits) -> TBits:
"""Return a copy of the bitstring."""
- pass
+ return self.__copy__()
- def __setattr__(self, attribute, value) ->None:
+ def __setattr__(self, attribute, value) -> None:
try:
+ # First try the ordinary attribute setter
super().__setattr__(attribute, value)
except AttributeError:
dtype = bitstring.dtypes.Dtype(attribute)
x = object.__new__(Bits)
if (set_fn := dtype.set_fn) is None:
- raise AttributeError(
- f"Cannot set attribute '{attribute}' as it does not have a set_fn."
- )
+ raise AttributeError(f"Cannot set attribute '{attribute}' as it does not have a set_fn.")
set_fn(x, value)
if len(x) != dtype.bitlength:
- raise CreationError(
- f"Can't initialise with value of length {len(x)} bits, as attribute has length of {dtype.bitlength} bits."
- )
+ raise CreationError(f"Can't initialise with value of length {len(x)} bits, "
+ f"as attribute has length of {dtype.bitlength} bits.")
self._bitstore = x._bitstore
return
- def __iadd__(self, bs: BitsType) ->BitArray:
+ def __iadd__(self, bs: BitsType) -> BitArray:
"""Append bs to current bitstring. Return self.
bs -- the bitstring to append.
@@ -142,20 +145,62 @@ class BitArray(Bits):
self._append(bs)
return self
- def __copy__(self) ->BitArray:
+ def __copy__(self) -> BitArray:
"""Return a new copy of the BitArray."""
s_copy = BitArray()
s_copy._bitstore = self._bitstore._copy()
assert s_copy._bitstore.immutable is False
return s_copy
- def __setitem__(self, key: Union[slice, int], value: BitsType) ->None:
+ def _setitem_int(self, key: int, value: Union[BitsType, int]) -> None:
+ if isinstance(value, numbers.Integral):
+ if value == 0:
+ self._bitstore[key] = 0
+ return
+ if value in (1, -1):
+ self._bitstore[key] = 1
+ return
+ raise ValueError(f"Cannot set a single bit with integer {value}.")
+ try:
+ value = self._create_from_bitstype(value)
+ except TypeError:
+ raise TypeError(f"Bitstring, integer or string expected. Got {type(value)}.")
+ positive_key = key + len(self) if key < 0 else key
+ if positive_key < 0 or positive_key >= len(self._bitstore):
+ raise IndexError(f"Bit position {key} out of range.")
+ self._bitstore[positive_key: positive_key + 1] = value._bitstore
+
+ def _setitem_slice(self, key: slice, value: BitsType) -> None:
+ if isinstance(value, numbers.Integral):
+ value = int(value)
+ if key.step not in [None, -1, 1]:
+ if value in [0, 1]:
+ self.set(value, range(*key.indices(len(self))))
+ return
+ else:
+ raise ValueError("Can't assign an integer except 0 or 1 to a slice with a step value.")
+ # To find the length we first get the slice
+ s = self._bitstore.getslice(key.start, key.stop)
+ length = len(s)
+ # Now create an int of the correct length
+ if value >= 0:
+ value = self.__class__(uint=value, length=length)
+ else:
+ value = self.__class__(int=value, length=length)
+ else:
+ try:
+ value = self._create_from_bitstype(value)
+ except TypeError:
+ raise TypeError(f"Bitstring, integer or string expected. Got {type(value)}.")
+ self._bitstore.__setitem__(key, value._bitstore)
+
+ def __setitem__(self, key: Union[slice, int], value: BitsType) -> None:
if isinstance(key, numbers.Integral):
self._setitem_int(int(key), value)
else:
self._setitem_slice(key, value)
- def __delitem__(self, key: Union[slice, int]) ->None:
+ def __delitem__(self, key: Union[slice, int]) -> None:
"""Delete item or range.
>>> a = BitArray('0x001122')
@@ -167,37 +212,37 @@ class BitArray(Bits):
self._bitstore.__delitem__(key)
return
- def __ilshift__(self: TBits, n: int) ->TBits:
+ def __ilshift__(self: TBits, n: int) -> TBits:
"""Shift bits by n to the left in place. Return self.
n -- the number of bits to shift. Must be >= 0.
"""
if n < 0:
- raise ValueError('Cannot shift by a negative amount.')
+ raise ValueError("Cannot shift by a negative amount.")
if not len(self):
- raise ValueError('Cannot shift an empty bitstring.')
+ raise ValueError("Cannot shift an empty bitstring.")
if not n:
return self
n = min(n, len(self))
return self._ilshift(n)
- def __irshift__(self: TBits, n: int) ->TBits:
+ def __irshift__(self: TBits, n: int) -> TBits:
"""Shift bits by n to the right in place. Return self.
n -- the number of bits to shift. Must be >= 0.
"""
if n < 0:
- raise ValueError('Cannot shift by a negative amount.')
+ raise ValueError("Cannot shift by a negative amount.")
if not len(self):
- raise ValueError('Cannot shift an empty bitstring.')
+ raise ValueError("Cannot shift an empty bitstring.")
if not n:
return self
n = min(n, len(self))
return self._irshift(n)
- def __imul__(self: TBits, n: int) ->TBits:
+ def __imul__(self: TBits, n: int) -> TBits:
"""Concatenate n copies of self in place. Return self.
Called for expressions of the form 'a *= 3'.
@@ -205,27 +250,57 @@ class BitArray(Bits):
"""
if n < 0:
- raise ValueError('Cannot multiply by a negative integer.')
+ raise ValueError("Cannot multiply by a negative integer.")
return self._imul(n)
- def __ior__(self: TBits, bs: BitsType) ->TBits:
+ def __ior__(self: TBits, bs: BitsType) -> TBits:
bs = self._create_from_bitstype(bs)
self._bitstore |= bs._bitstore
return self
- def __iand__(self: TBits, bs: BitsType) ->TBits:
+ def __iand__(self: TBits, bs: BitsType) -> TBits:
bs = self._create_from_bitstype(bs)
self._bitstore &= bs._bitstore
return self
- def __ixor__(self: TBits, bs: BitsType) ->TBits:
+ def __ixor__(self: TBits, bs: BitsType) -> TBits:
bs = self._create_from_bitstype(bs)
self._bitstore ^= bs._bitstore
return self
- def replace(self, old: BitsType, new: BitsType, start: Optional[int]=
- None, end: Optional[int]=None, count: Optional[int]=None,
- bytealigned: Optional[bool]=None) ->int:
+ def _replace(self, old: Bits, new: Bits, start: int, end: int, count: int, bytealigned: Optional[bool]) -> int:
+ if bytealigned is None:
+ bytealigned = bitstring.options.bytealigned
+ # First find all the places where we want to do the replacements
+ starting_points: List[int] = []
+ for x in self.findall(old, start, end, bytealigned=bytealigned):
+ if not starting_points:
+ starting_points.append(x)
+ elif x >= starting_points[-1] + len(old):
+ # Can only replace here if it hasn't already been replaced!
+ starting_points.append(x)
+ if count != 0 and len(starting_points) == count:
+ break
+ if not starting_points:
+ return 0
+ replacement_list = [self._bitstore.getslice(0, starting_points[0])]
+ for i in range(len(starting_points) - 1):
+ replacement_list.append(new._bitstore)
+ replacement_list.append(
+ self._bitstore.getslice(starting_points[i] + len(old), starting_points[i + 1]))
+ # Final replacement
+ replacement_list.append(new._bitstore)
+ replacement_list.append(self._bitstore.getslice(starting_points[-1] + len(old), None))
+ if bitstring.options.lsb0:
+ # Addition of bitarray is always on the right, so assemble from other end
+ replacement_list.reverse()
+ self._bitstore.clear()
+ for r in replacement_list:
+ self._bitstore += r
+ return len(starting_points)
+
+ def replace(self, old: BitsType, new: BitsType, start: Optional[int] = None, end: Optional[int] = None,
+ count: Optional[int] = None, bytealigned: Optional[bool] = None) -> int:
"""Replace all occurrences of old with new in place.
Returns number of replacements made.
@@ -245,9 +320,20 @@ class BitArray(Bits):
out of range.
"""
- pass
-
- def insert(self, bs: BitsType, pos: int) ->None:
+ if count == 0:
+ return 0
+ old = self._create_from_bitstype(old)
+ new = self._create_from_bitstype(new)
+ if len(old) == 0:
+ raise ValueError("Empty bitstring cannot be replaced.")
+ start, end = self._validate_slice(start, end)
+
+ if new is self:
+ # Prevent self assignment woes
+ new = copy.copy(self)
+ return self._replace(old, new, start, end, 0 if count is None else count, bytealigned)
+
+ def insert(self, bs: BitsType, pos: int) -> None:
"""Insert bs at bit position pos.
bs -- The bitstring to insert.
@@ -256,9 +342,18 @@ class BitArray(Bits):
Raises ValueError if pos < 0 or pos > len(self).
"""
- pass
-
- def overwrite(self, bs: BitsType, pos: int) ->None:
+ bs = self._create_from_bitstype(bs)
+ if len(bs) == 0:
+ return
+ if bs is self:
+ bs = self._copy()
+ if pos < 0:
+ pos += len(self)
+ if not 0 <= pos <= len(self):
+ raise ValueError("Invalid insert position.")
+ self._insert(bs, pos)
+
+ def overwrite(self, bs: BitsType, pos: int) -> None:
"""Overwrite with bs at bit position pos.
bs -- The bitstring to overwrite with.
@@ -267,26 +362,39 @@ class BitArray(Bits):
Raises ValueError if pos < 0 or pos > len(self).
"""
- pass
+ bs = self._create_from_bitstype(bs)
+ if len(bs) == 0:
+ return
+ if pos < 0:
+ pos += len(self)
+ if pos < 0 or pos > len(self):
+ raise ValueError("Overwrite starts outside boundary of bitstring.")
+ self._overwrite(bs, pos)
- def append(self, bs: BitsType) ->None:
+ def append(self, bs: BitsType) -> None:
"""Append a bitstring to the current bitstring.
bs -- The bitstring to append.
"""
- pass
+ self._append(bs)
- def prepend(self, bs: BitsType) ->None:
+ def prepend(self, bs: BitsType) -> None:
"""Prepend a bitstring to the current bitstring.
bs -- The bitstring to prepend.
"""
- pass
+ self._prepend(bs)
+
+ def _append_msb0(self, bs: BitsType) -> None:
+ self._addright(self._create_from_bitstype(bs))
- def reverse(self, start: Optional[int]=None, end: Optional[int]=None
- ) ->None:
+ def _append_lsb0(self, bs: BitsType) -> None:
+ bs = self._create_from_bitstype(bs)
+ self._addleft(bs)
+
+ def reverse(self, start: Optional[int] = None, end: Optional[int] = None) -> None:
"""Reverse bits in-place.
start -- Position of first bit to reverse. Defaults to 0.
@@ -298,10 +406,15 @@ class BitArray(Bits):
Raises ValueError if start < 0, end > len(self) or end < start.
"""
- pass
+ start, end = self._validate_slice(start, end)
+ if start == 0 and end == len(self):
+ self._bitstore.reverse()
+ return
+ s = self._slice(start, end)
+ s._bitstore.reverse()
+ self[start:end] = s
- def set(self, value: Any, pos: Optional[Union[int, Iterable[int]]]=None
- ) ->None:
+ def set(self, value: Any, pos: Optional[Union[int, Iterable[int]]] = None) -> None:
"""Set one or many bits to 1 or 0.
value -- If bool(value) is True bits are set to 1, otherwise they are set to 0.
@@ -312,9 +425,20 @@ class BitArray(Bits):
Raises IndexError if pos < -len(self) or pos >= len(self).
"""
- pass
+ if pos is None:
+ # Set all bits to either 1 or 0
+ self._setint(-1 if value else 0)
+ return
+ if not isinstance(pos, abc.Iterable):
+ pos = (pos,)
+ v = 1 if value else 0
+ if isinstance(pos, range):
+ self._bitstore.__setitem__(slice(pos.start, pos.stop, pos.step), v)
+ return
+ for p in pos:
+ self._bitstore[p] = v
- def invert(self, pos: Optional[Union[Iterable[int], int]]=None) ->None:
+ def invert(self, pos: Optional[Union[Iterable[int], int]] = None) -> None:
"""Invert one or many bits from 0 to 1 or vice versa.
pos -- Either a single bit position or an iterable of bit positions.
@@ -323,10 +447,21 @@ class BitArray(Bits):
Raises IndexError if pos < -len(self) or pos >= len(self).
"""
- pass
-
- def ror(self, bits: int, start: Optional[int]=None, end: Optional[int]=None
- ) ->None:
+ if pos is None:
+ self._invert_all()
+ return
+ if not isinstance(pos, abc.Iterable):
+ pos = (pos,)
+ length = len(self)
+
+ for p in pos:
+ if p < 0:
+ p += length
+ if not 0 <= p < length:
+ raise IndexError(f"Bit position {p} out of range.")
+ self._invert(p)
+
+ def ror(self, bits: int, start: Optional[int] = None, end: Optional[int] = None) -> None:
"""Rotate bits to the right in-place.
bits -- The number of bits to rotate by.
@@ -336,10 +471,22 @@ class BitArray(Bits):
Raises ValueError if bits < 0.
"""
- pass
+ if not len(self):
+ raise Error("Cannot rotate an empty bitstring.")
+ if bits < 0:
+ raise ValueError("Cannot rotate by negative amount.")
+ self._ror(bits, start, end)
+
+ def _ror_msb0(self, bits: int, start: Optional[int] = None, end: Optional[int] = None) -> None:
+ start, end = self._validate_slice(start, end) # the _slice deals with msb0/lsb0
+ bits %= (end - start)
+ if not bits:
+ return
+ rhs = self._slice(end - bits, end)
+ self._delete(bits, end - bits)
+ self._insert(rhs, start)
- def rol(self, bits: int, start: Optional[int]=None, end: Optional[int]=None
- ) ->None:
+ def rol(self, bits: int, start: Optional[int] = None, end: Optional[int] = None) -> None:
"""Rotate bits to the left in-place.
bits -- The number of bits to rotate by.
@@ -349,11 +496,23 @@ class BitArray(Bits):
Raises ValueError if bits < 0.
"""
- pass
+ if not len(self):
+ raise Error("Cannot rotate an empty bitstring.")
+ if bits < 0:
+ raise ValueError("Cannot rotate by negative amount.")
+ self._rol(bits, start, end)
+
+ def _rol_msb0(self, bits: int, start: Optional[int] = None, end: Optional[int] = None):
+ start, end = self._validate_slice(start, end)
+ bits %= (end - start)
+ if bits == 0:
+ return
+ lhs = self._slice(start, start + bits)
+ self._delete(bits, start)
+ self._insert(lhs, end - bits)
- def byteswap(self, fmt: Optional[Union[int, Iterable[int], str]]=None,
- start: Optional[int]=None, end: Optional[int]=None, repeat: bool=True
- ) ->int:
+ def byteswap(self, fmt: Optional[Union[int, Iterable[int], str]] = None, start: Optional[int] = None,
+ end: Optional[int] = None, repeat: bool = True) -> int:
"""Change the endianness in-place. Return number of repeats of fmt done.
fmt -- A compact structure string, an integer number of bytes or
@@ -365,8 +524,53 @@ class BitArray(Bits):
as much as possible.
"""
- pass
-
- def clear(self) ->None:
+ start_v, end_v = self._validate_slice(start, end)
+ if fmt is None or fmt == 0:
+ # reverse all of the whole bytes.
+ bytesizes = [(end_v - start_v) // 8]
+ elif isinstance(fmt, numbers.Integral):
+ if fmt < 0:
+ raise ValueError(f"Improper byte length {fmt}.")
+ bytesizes = [fmt]
+ elif isinstance(fmt, str):
+ if not (m := utils.BYTESWAP_STRUCT_PACK_RE.match(fmt)):
+ raise ValueError(f"Cannot parse format string {fmt}.")
+ # Split the format string into a list of 'q', '4h' etc.
+ formatlist = re.findall(utils.STRUCT_SPLIT_RE, m.group('fmt'))
+ # Now deal with multiplicative factors, 4h -> hhhh etc.
+ bytesizes = []
+ for f in formatlist:
+ if len(f) == 1:
+ bytesizes.append(utils.PACK_CODE_SIZE[f])
+ else:
+ bytesizes.extend([utils.PACK_CODE_SIZE[f[-1]]] * int(f[:-1]))
+ elif isinstance(fmt, abc.Iterable):
+ bytesizes = fmt
+ for bytesize in bytesizes:
+ if not isinstance(bytesize, numbers.Integral) or bytesize < 0:
+ raise ValueError(f"Improper byte length {bytesize}.")
+ else:
+ raise TypeError("Format must be an integer, string or iterable.")
+
+ repeats = 0
+ totalbitsize: int = 8 * sum(bytesizes)
+ if not totalbitsize:
+ return 0
+ if repeat:
+ # Try to repeat up to the end of the bitstring.
+ finalbit = end_v
+ else:
+ # Just try one (set of) byteswap(s).
+ finalbit = start_v + totalbitsize
+ for patternend in range(start_v + totalbitsize, finalbit + 1, totalbitsize):
+ bytestart = patternend - totalbitsize
+ for bytesize in bytesizes:
+ byteend = bytestart + bytesize * 8
+ self._reversebytes(bytestart, byteend)
+ bytestart += bytesize * 8
+ repeats += 1
+ return repeats
+
+ def clear(self) -> None:
"""Remove all bits, reset to zero length."""
- pass
+ self._clear()
diff --git a/bitstring/bits.py b/bitstring/bits.py
index f6d7d08..6893709 100644
--- a/bitstring/bits.py
+++ b/bitstring/bits.py
@@ -1,4 +1,5 @@
from __future__ import annotations
+
import numbers
import pathlib
import sys
@@ -18,9 +19,13 @@ from bitstring.dtypes import Dtype, dtype_register
from bitstring.fp8 import p4binary_fmt, p3binary_fmt
from bitstring.mxfp import e3m2mxfp_fmt, e2m3mxfp_fmt, e2m1mxfp_fmt, e4m3mxfp_saturate_fmt, e5m2mxfp_saturate_fmt
from bitstring.bitstring_options import Colour
-BitsType = Union['Bits', str, Iterable[Any], bool, BinaryIO, bytearray,
- bytes, memoryview, bitarray.bitarray]
-TBits = TypeVar('TBits', bound='Bits')
+
+# Things that can be converted to Bits when a Bits type is needed
+BitsType = Union['Bits', str, Iterable[Any], bool, BinaryIO, bytearray, bytes, memoryview, bitarray.bitarray]
+
+TBits = TypeVar("TBits", bound='Bits')
+
+# Maximum number of digits to use in __str__ and __repr__.
MAX_CHARS: int = 250
@@ -61,10 +66,10 @@ class Bits:
len -- Length of the bitstring in bits.
"""
- __slots__ = '_bitstore', '_filename'
+ __slots__ = ('_bitstore', '_filename')
- def __init__(self, auto: Optional[Union[BitsType, int]]=None, /, length:
- Optional[int]=None, offset: Optional[int]=None, **kwargs) ->None:
+ def __init__(self, auto: Optional[Union[BitsType, int]] = None, /, length: Optional[int] = None,
+ offset: Optional[int] = None, **kwargs) -> None:
"""Either specify an 'auto' initialiser:
A string of comma separated tokens, an integer, a file object,
a bytearray, a boolean iterable, an array or another bitstring.
@@ -105,11 +110,11 @@ class Bits:
"""
self._bitstore.immutable = True
- def __new__(cls: Type[TBits], auto: Optional[Union[BitsType, int]]=None,
- /, length: Optional[int]=None, offset: Optional[int]=None, pos:
- Optional[int]=None, **kwargs) ->TBits:
+ def __new__(cls: Type[TBits], auto: Optional[Union[BitsType, int]] = None, /, length: Optional[int] = None,
+ offset: Optional[int] = None, pos: Optional[int] = None, **kwargs) -> TBits:
x = super().__new__(cls)
if auto is None and not kwargs:
+ # No initialiser so fill with zero bits up to length
if length is not None:
x._bitstore = BitStore(length)
x._bitstore.setall(0)
@@ -119,39 +124,80 @@ class Bits:
x._initialise(auto, length, offset, **kwargs)
return x
- def __getattr__(self, attribute: str) ->Any:
+ @classmethod
+ def _create_from_bitstype(cls: Type[TBits], auto: BitsType, /) -> TBits:
+ if isinstance(auto, cls):
+ return auto
+ b = super().__new__(cls)
+ b._setauto_no_length_or_offset(auto)
+ return b
+
+ def _initialise(self, auto: Any, /, length: Optional[int], offset: Optional[int], **kwargs) -> None:
+ if auto is not None:
+ if isinstance(auto, numbers.Integral):
+ # Initialise with s zero bits.
+ if auto < 0:
+ raise bitstring.CreationError(f"Can't create bitstring of negative length {auto}.")
+ self._bitstore = BitStore(int(auto))
+ self._bitstore.setall(0)
+ return
+ self._setauto(auto, length, offset)
+ return
+ k, v = kwargs.popitem()
+ if k == 'bytes':
+ # Special case for bytes as we want to allow offsets and lengths to work only on creation.
+ self._setbytes_with_truncation(v, length, offset)
+ return
+ if k == 'filename':
+ self._setfile(v, length, offset)
+ return
+ if k == 'bitarray':
+ self._setbitarray(v, length, offset)
+ return
+ if k == 'auto':
+ raise bitstring.CreationError(
+ f"The 'auto' parameter should not be given explicitly - just use the first positional argument. "
+ f"Instead of '{self.__class__.__name__}(auto=x)' use '{self.__class__.__name__}(x)'.")
+ if offset is not None:
+ raise bitstring.CreationError("offset cannot be used when initialising with '{k}'.")
+ try:
+ Dtype(k, length).set_fn(self, v)
+ except ValueError as e:
+ raise bitstring.CreationError(e)
+
+ def __getattr__(self, attribute: str) -> Any:
+ # Support for arbitrary attributes like u16 or f64.
try:
d = Dtype(attribute)
except ValueError:
- raise AttributeError(
- f"'{self.__class__.__name__}' object has no attribute '{attribute}'."
- )
+ raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{attribute}'.")
if d.bitlength is not None and len(self) != d.bitlength:
- raise ValueError(
- f"bitstring length {len(self)} doesn't match length {d.bitlength} of property '{attribute}'."
- )
+ raise ValueError(f"bitstring length {len(self)} doesn't match length {d.bitlength} of property '{attribute}'.")
return d.get_fn(self)
- def __iter__(self) ->Iterable[bool]:
+ def __iter__(self) -> Iterable[bool]:
return iter(self._bitstore)
- def __copy__(self: TBits) ->TBits:
+ def __copy__(self: TBits) -> TBits:
"""Return a new copy of the Bits for the copy module."""
+ # Note that if you want a new copy (different ID), use _copy instead.
+ # The copy can return self as it's immutable.
return self
- def __lt__(self, other: Any) ->bool:
+ def __lt__(self, other: Any) -> bool:
+ # bitstrings can't really be ordered.
return NotImplemented
- def __gt__(self, other: Any) ->bool:
+ def __gt__(self, other: Any) -> bool:
return NotImplemented
- def __le__(self, other: Any) ->bool:
+ def __le__(self, other: Any) -> bool:
return NotImplemented
- def __ge__(self, other: Any) ->bool:
+ def __ge__(self, other: Any) -> bool:
return NotImplemented
- def __add__(self: TBits, bs: BitsType) ->TBits:
+ def __add__(self: TBits, bs: BitsType) -> TBits:
"""Concatenate bitstrings and return new bitstring.
bs -- the bitstring to append.
@@ -165,7 +211,7 @@ class Bits:
s._addleft(self)
return s
- def __radd__(self: TBits, bs: BitsType) ->TBits:
+ def __radd__(self: TBits, bs: BitsType) -> TBits:
"""Append current bitstring to bs and return new bitstring.
bs -- An object that can be 'auto' initialised as a bitstring that will be appended to.
@@ -175,15 +221,14 @@ class Bits:
return bs.__add__(self)
@overload
- def __getitem__(self: TBits, key: slice, /) ->TBits:
+ def __getitem__(self: TBits, key: slice, /) -> TBits:
...
@overload
- def __getitem__(self, key: int, /) ->bool:
+ def __getitem__(self, key: int, /) -> bool:
...
- def __getitem__(self: TBits, key: Union[slice, int], /) ->Union[TBits, bool
- ]:
+ def __getitem__(self: TBits, key: Union[slice, int], /) -> Union[TBits, bool]:
"""Return a new bitstring representing a slice of the current bitstring.
Indices are in units of the step parameter (default 1 bit).
@@ -201,14 +246,14 @@ class Bits:
bs._bitstore = self._bitstore.getslice_withstep(key)
return bs
- def __len__(self) ->int:
+ def __len__(self) -> int:
"""Return the length of the bitstring in bits."""
return self._getlength()
- def __bytes__(self) ->bytes:
+ def __bytes__(self) -> bytes:
return self.tobytes()
- def __str__(self) ->str:
+ def __str__(self) -> str:
"""Return approximate string representation of bitstring for printing.
Short strings will be given wholly in hexadecimal or binary. Longer
@@ -220,16 +265,32 @@ class Bits:
if not length:
return ''
if length > MAX_CHARS * 4:
- return ''.join(('0x', self[0:MAX_CHARS * 4]._gethex(), '...'))
+ # Too long for hex. Truncate...
+ return ''.join(('0x', self[0:MAX_CHARS*4]._gethex(), '...'))
+ # If it's quite short and we can't do hex then use bin
if length < 32 and length % 4 != 0:
return '0b' + self.bin
+ # If we can use hex then do so
if not length % 4:
return '0x' + self.hex
+ # Otherwise first we do as much as we can in hex
+ # then add on 1, 2 or 3 bits on at the end
bits_at_end = length % 4
- return ''.join(('0x', self[0:length - bits_at_end]._gethex(), ', ',
- '0b', self[length - bits_at_end:]._getbin()))
+ return ''.join(('0x', self[0:length - bits_at_end]._gethex(),
+ ', ', '0b', self[length - bits_at_end:]._getbin()))
+
+ def _repr(self, classname: str, length: int, pos: int):
+ pos_string = f', pos={pos}' if pos else ''
+ if hasattr(self, '_filename') and self._filename:
+ return f"{classname}(filename={self._filename!r}, length={length}{pos_string})"
+ else:
+ s = self.__str__()
+ lengthstring = ''
+ if s.endswith('...'):
+ lengthstring = f' # length={length}'
+ return f"{classname}('{s}'{pos_string}){lengthstring}"
- def __repr__(self) ->str:
+ def __repr__(self) -> str:
"""Return representation that could be used to recreate the bitstring.
If the returned string is too long it will be truncated. See __str__().
@@ -237,7 +298,7 @@ class Bits:
"""
return self._repr(self.__class__.__name__, len(self), 0)
- def __eq__(self, bs: Any, /) ->bool:
+ def __eq__(self, bs: Any, /) -> bool:
"""Return True if two bitstrings have the same binary representation.
>>> BitArray('0b1110') == '0xe'
@@ -249,7 +310,7 @@ class Bits:
except TypeError:
return False
- def __ne__(self, bs: Any, /) ->bool:
+ def __ne__(self, bs: Any, /) -> bool:
"""Return False if two bitstrings have the same binary representation.
>>> BitArray('0b111') == '0x7'
@@ -258,43 +319,43 @@ class Bits:
"""
return not self.__eq__(bs)
- def __invert__(self: TBits) ->TBits:
+ def __invert__(self: TBits) -> TBits:
"""Return bitstring with every bit inverted.
Raises Error if the bitstring is empty.
"""
if len(self) == 0:
- raise bitstring.Error('Cannot invert empty bitstring.')
+ raise bitstring.Error("Cannot invert empty bitstring.")
s = self._copy()
s._invert_all()
return s
- def __lshift__(self: TBits, n: int, /) ->TBits:
+ def __lshift__(self: TBits, n: int, /) -> TBits:
"""Return bitstring with bits shifted by n to the left.
n -- the number of bits to shift. Must be >= 0.
"""
if n < 0:
- raise ValueError('Cannot shift by a negative amount.')
+ raise ValueError("Cannot shift by a negative amount.")
if len(self) == 0:
- raise ValueError('Cannot shift an empty bitstring.')
+ raise ValueError("Cannot shift an empty bitstring.")
n = min(n, len(self))
s = self._absolute_slice(n, len(self))
s._addright(Bits(n))
return s
- def __rshift__(self: TBits, n: int, /) ->TBits:
+ def __rshift__(self: TBits, n: int, /) -> TBits:
"""Return bitstring with bits shifted by n to the right.
n -- the number of bits to shift. Must be >= 0.
"""
if n < 0:
- raise ValueError('Cannot shift by a negative amount.')
+ raise ValueError("Cannot shift by a negative amount.")
if len(self) == 0:
- raise ValueError('Cannot shift an empty bitstring.')
+ raise ValueError("Cannot shift an empty bitstring.")
if not n:
return self._copy()
s = self.__class__(length=min(n, len(self)))
@@ -302,7 +363,7 @@ class Bits:
s._addright(self._absolute_slice(0, len(self) - n))
return s
- def __mul__(self: TBits, n: int, /) ->TBits:
+ def __mul__(self: TBits, n: int, /) -> TBits:
"""Return bitstring consisting of n concatenations of self.
Called for expression of the form 'a = b*3'.
@@ -310,14 +371,14 @@ class Bits:
"""
if n < 0:
- raise ValueError('Cannot multiply by a negative integer.')
+ raise ValueError("Cannot multiply by a negative integer.")
if not n:
return self.__class__()
s = self._copy()
s._imul(n)
return s
- def __rmul__(self: TBits, n: int, /) ->TBits:
+ def __rmul__(self: TBits, n: int, /) -> TBits:
"""Return bitstring consisting of n concatenations of self.
Called for expressions of the form 'a = 3*b'.
@@ -326,7 +387,7 @@ class Bits:
"""
return self.__mul__(n)
- def __and__(self: TBits, bs: BitsType, /) ->TBits:
+ def __and__(self: TBits, bs: BitsType, /) -> TBits:
"""Bit-wise 'and' between two bitstrings. Returns new bitstring.
bs -- The bitstring to '&' with.
@@ -341,7 +402,7 @@ class Bits:
s._bitstore = self._bitstore & bs._bitstore
return s
- def __rand__(self: TBits, bs: BitsType, /) ->TBits:
+ def __rand__(self: TBits, bs: BitsType, /) -> TBits:
"""Bit-wise 'and' between two bitstrings. Returns new bitstring.
bs -- the bitstring to '&' with.
@@ -351,7 +412,7 @@ class Bits:
"""
return self.__and__(bs)
- def __or__(self: TBits, bs: BitsType, /) ->TBits:
+ def __or__(self: TBits, bs: BitsType, /) -> TBits:
"""Bit-wise 'or' between two bitstrings. Returns new bitstring.
bs -- The bitstring to '|' with.
@@ -366,7 +427,7 @@ class Bits:
s._bitstore = self._bitstore | bs._bitstore
return s
- def __ror__(self: TBits, bs: BitsType, /) ->TBits:
+ def __ror__(self: TBits, bs: BitsType, /) -> TBits:
"""Bit-wise 'or' between two bitstrings. Returns new bitstring.
bs -- The bitstring to '|' with.
@@ -376,7 +437,7 @@ class Bits:
"""
return self.__or__(bs)
- def __xor__(self: TBits, bs: BitsType, /) ->TBits:
+ def __xor__(self: TBits, bs: BitsType, /) -> TBits:
"""Bit-wise 'xor' between two bitstrings. Returns new bitstring.
bs -- The bitstring to '^' with.
@@ -389,7 +450,7 @@ class Bits:
s._bitstore = self._bitstore ^ bs._bitstore
return s
- def __rxor__(self: TBits, bs: BitsType, /) ->TBits:
+ def __rxor__(self: TBits, bs: BitsType, /) -> TBits:
"""Bit-wise 'xor' between two bitstrings. Returns new bitstring.
bs -- The bitstring to '^' with.
@@ -399,7 +460,7 @@ class Bits:
"""
return self.__xor__(bs)
- def __contains__(self, bs: BitsType, /) ->bool:
+ def __contains__(self, bs: BitsType, /) -> bool:
"""Return whether bs is contained in the current bitstring.
bs -- The bitstring to search for.
@@ -408,125 +469,416 @@ class Bits:
found = Bits.find(self, bs, bytealigned=False)
return bool(found)
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
"""Return an integer hash of the object."""
+ # Only requirement is that equal bitstring should return the same hash.
+ # For equal bitstrings the bytes at the start/end will be the same and they will have the same length
+ # (need to check the length as there could be zero padding when getting the bytes). We do not check any
+ # bit position inside the bitstring as that does not feature in the __eq__ operation.
if len(self) <= 2000:
+ # Use the whole bitstring.
return hash((self.tobytes(), len(self)))
else:
+ # We can't in general hash the whole bitstring (it could take hours!)
+ # So instead take some bits from the start and end.
return hash(((self[:800] + self[-800:]).tobytes(), len(self)))
- def __bool__(self) ->bool:
+ def __bool__(self) -> bool:
"""Return False if bitstring is empty, otherwise return True."""
return len(self) != 0
- def _clear(self) ->None:
+ def _clear(self) -> None:
"""Reset the bitstring to an empty state."""
- pass
+ self._bitstore = BitStore()
- def _setauto_no_length_or_offset(self, s: BitsType, /) ->None:
+ def _setauto_no_length_or_offset(self, s: BitsType, /) -> None:
"""Set bitstring from a bitstring, file, bool, array, iterable or string."""
- pass
+ if isinstance(s, str):
+ self._bitstore = bitstore_helpers.str_to_bitstore(s)
+ elif isinstance(s, Bits):
+ self._bitstore = s._bitstore.copy()
+ elif isinstance(s, (bytes, bytearray, memoryview)):
+ self._bitstore = BitStore.frombytes(bytearray(s))
+ elif isinstance(s, io.BytesIO):
+ self._bitstore = BitStore.frombytes(s.getvalue())
+ elif isinstance(s, io.BufferedReader):
+ self._setfile(s.name)
+ elif isinstance(s, bitarray.bitarray):
+ self._bitstore = BitStore(s)
+ elif isinstance(s, array.array):
+ self._bitstore = BitStore.frombytes(s.tobytes())
+ elif isinstance(s, abc.Iterable):
+ # Evaluate each item as True or False and set bits to 1 or 0.
+ self._setbin_unsafe(''.join(str(int(bool(x))) for x in s))
+ elif isinstance(s, numbers.Integral):
+ raise TypeError(f"It's no longer possible to auto initialise a bitstring from an integer."
+ f" Use '{self.__class__.__name__}({s})' instead of just '{s}' as this makes it "
+ f"clearer that a bitstring of {int(s)} zero bits will be created.")
+ else:
+ raise TypeError(f"Cannot initialise bitstring from type '{type(s)}'.")
- def _setauto(self, s: BitsType, length: Optional[int], offset: Optional
- [int], /) ->None:
+ def _setauto(self, s: BitsType, length: Optional[int], offset: Optional[int], /) -> None:
"""Set bitstring from a bitstring, file, bool, array, iterable or string."""
- pass
-
- def _setfile(self, filename: str, length: Optional[int]=None, offset:
- Optional[int]=None) ->None:
+ # As s can be so many different things it's important to do the checks
+ # in the correct order, as some types are also other allowed types.
+ if offset is None and length is None:
+ return self._setauto_no_length_or_offset(s)
+ if offset is None:
+ offset = 0
+
+ if isinstance(s, io.BytesIO):
+ if length is None:
+ length = s.seek(0, 2) * 8 - offset
+ byteoffset, offset = divmod(offset, 8)
+ bytelength = (length + byteoffset * 8 + offset + 7) // 8 - byteoffset
+ if length + byteoffset * 8 + offset > s.seek(0, 2) * 8:
+ raise bitstring.CreationError("BytesIO object is not long enough for specified length and offset.")
+ self._bitstore = BitStore.frombytes(s.getvalue()[byteoffset: byteoffset + bytelength]).getslice(
+ offset, offset + length)
+ return
+
+ if isinstance(s, io.BufferedReader):
+ self._setfile(s.name, length, offset)
+ return
+
+ if isinstance(s, (str, Bits, bytes, bytearray, memoryview, io.BytesIO, io.BufferedReader,
+ bitarray.bitarray, array.array, abc.Iterable)):
+ raise bitstring.CreationError(f"Cannot initialise bitstring from type '{type(s)}' when using explicit lengths or offsets.")
+ raise TypeError(f"Cannot initialise bitstring from type '{type(s)}'.")
+
+ def _setfile(self, filename: str, length: Optional[int] = None, offset: Optional[int] = None) -> None:
"""Use file as source of bits."""
- pass
+ with open(pathlib.Path(filename), 'rb') as source:
+ if offset is None:
+ offset = 0
+ m = mmap.mmap(source.fileno(), 0, access=mmap.ACCESS_READ)
+ if offset == 0:
+ self._filename = source.name
+ self._bitstore = BitStore.frombuffer(m, length=length)
+ else:
+ # If offset is given then always read into memory.
+ temp = BitStore.frombuffer(m)
+ if length is None:
+ if offset > len(temp):
+ raise bitstring.CreationError(f"The offset of {offset} bits is greater than the file length ({len(temp)} bits).")
+ self._bitstore = temp.getslice(offset, None)
+ else:
+ self._bitstore = temp.getslice(offset, offset + length)
+ if len(self) != length:
+ raise bitstring.CreationError(f"Can't use a length of {length} bits and an offset of {offset} bits as file length is only {len(temp)} bits.")
+
+ def _setbitarray(self, ba: bitarray.bitarray, length: Optional[int], offset: Optional[int]) -> None:
+ if offset is None:
+ offset = 0
+ if offset > len(ba):
+ raise bitstring.CreationError(f"Offset of {offset} too large for bitarray of length {len(ba)}.")
+ if length is None:
+ self._bitstore = BitStore(ba[offset:])
+ else:
+ if offset + length > len(ba):
+ raise bitstring.CreationError(
+ f"Offset of {offset} and length of {length} too large for bitarray of length {len(ba)}.")
+ self._bitstore = BitStore(ba[offset: offset + length])
- def _setbytes(self, data: Union[bytearray, bytes, List], length: None=None
- ) ->None:
+ def _setbits(self, bs: BitsType, length: None = None) -> None:
+ bs = Bits._create_from_bitstype(bs)
+ self._bitstore = bs._bitstore
+
+ def _setp3binary(self, f: float) -> None:
+ self._bitstore = bitstore_helpers.p3binary2bitstore(f)
+
+ def _setp4binary(self, f: float) -> None:
+ self._bitstore = bitstore_helpers.p4binary2bitstore(f)
+
+ def _sete4m3mxfp(self, f: float) -> None:
+ self._bitstore = bitstore_helpers.e4m3mxfp2bitstore(f)
+
+ def _sete5m2mxfp(self, f: float) -> None:
+ self._bitstore = bitstore_helpers.e5m2mxfp2bitstore(f)
+
+ def _sete3m2mxfp(self, f: float) -> None:
+ self._bitstore = bitstore_helpers.e3m2mxfp2bitstore(f)
+
+ def _sete2m3mxfp(self, f: float) -> None:
+ self._bitstore = bitstore_helpers.e2m3mxfp2bitstore(f)
+
+ def _sete2m1mxfp(self, f: float) -> None:
+ self._bitstore = bitstore_helpers.e2m1mxfp2bitstore(f)
+
+ def _sete8m0mxfp(self, f: float) -> None:
+ self._bitstore = bitstore_helpers.e8m0mxfp2bitstore(f)
+
+ def _setmxint(self, f: float) -> None:
+ self._bitstore = bitstore_helpers.mxint2bitstore(f)
+
+ def _setbytes(self, data: Union[bytearray, bytes, List], length:None = None) -> None:
"""Set the data from a bytes or bytearray object."""
- pass
+ self._bitstore = BitStore.frombytes(bytes(data))
- def _setbytes_with_truncation(self, data: Union[bytearray, bytes],
- length: Optional[int]=None, offset: Optional[int]=None) ->None:
+ def _setbytes_with_truncation(self, data: Union[bytearray, bytes], length: Optional[int] = None, offset: Optional[int] = None) -> None:
"""Set the data from a bytes or bytearray object, with optional offset and length truncations."""
- pass
+ if offset is None and length is None:
+ return self._setbytes(data)
+ data = bytearray(data)
+ if offset is None:
+ offset = 0
+ if length is None:
+ # Use to the end of the data
+ length = len(data) * 8 - offset
+ else:
+ if length + offset > len(data) * 8:
+ raise bitstring.CreationError(f"Not enough data present. Need {length + offset} bits, have {len(data) * 8}.")
+ self._bitstore = BitStore.frombytes(data).getslice_msb0(offset, offset + length)
- def _getbytes(self) ->bytes:
+ def _getbytes(self) -> bytes:
"""Return the data as an ordinary bytes object."""
- pass
- _unprintable = list(range(0, 32))
- _unprintable.extend(range(127, 255))
+ if len(self) % 8:
+ raise bitstring.InterpretError("Cannot interpret as bytes unambiguously - not multiple of 8 bits.")
+ return self._bitstore.tobytes()
+
+ _unprintable = list(range(0x00, 0x20)) # ASCII control characters
+ _unprintable.extend(range(0x7f, 0xff)) # DEL char + non-ASCII
- def _getbytes_printable(self) ->str:
+ def _getbytes_printable(self) -> str:
"""Return an approximation of the data as a string of printable characters."""
- pass
+ bytes_ = self._getbytes()
+ # For everything that isn't printable ASCII, use value from 'Latin Extended-A' unicode block.
+ string = ''.join(chr(0x100 + x) if x in Bits._unprintable else chr(x) for x in bytes_)
+ return string
- def _setuint(self, uint: int, length: Optional[int]=None) ->None:
+ def _setuint(self, uint: int, length: Optional[int] = None) -> None:
"""Reset the bitstring to have given unsigned int interpretation."""
- pass
-
- def _getuint(self) ->int:
+ # If no length given, and we've previously been given a length, use it.
+ if length is None and hasattr(self, 'len') and len(self) != 0:
+ length = len(self)
+ if length is None or length == 0:
+ raise bitstring.CreationError("A non-zero length must be specified with a uint initialiser.")
+ self._bitstore = bitstore_helpers.int2bitstore(uint, length, False)
+
+ def _getuint(self) -> int:
"""Return data as an unsigned int."""
- pass
+ if len(self) == 0:
+ raise bitstring.InterpretError("Cannot interpret a zero length bitstring as an integer.")
+ return self._bitstore.slice_to_uint()
- def _setint(self, int_: int, length: Optional[int]=None) ->None:
+ def _setint(self, int_: int, length: Optional[int] = None) -> None:
"""Reset the bitstring to have given signed int interpretation."""
- pass
-
- def _getint(self) ->int:
+ # If no length given, and we've previously been given a length, use it.
+ if length is None and hasattr(self, 'len') and len(self) != 0:
+ length = len(self)
+ if length is None or length == 0:
+ raise bitstring.CreationError("A non-zero length must be specified with an int initialiser.")
+ self._bitstore = bitstore_helpers.int2bitstore(int_, length, True)
+
+ def _getint(self) -> int:
"""Return data as a two's complement signed int."""
- pass
+ if len(self) == 0:
+ raise bitstring.InterpretError("Cannot interpret bitstring without a length as an integer.")
+ return self._bitstore.slice_to_int()
- def _setuintbe(self, uintbe: int, length: Optional[int]=None) ->None:
+ def _setuintbe(self, uintbe: int, length: Optional[int] = None) -> None:
"""Set the bitstring to a big-endian unsigned int interpretation."""
- pass
+ if length is None and hasattr(self, 'len') and len(self) != 0:
+ length = len(self)
+ if length is None or length == 0:
+ raise bitstring.CreationError("A non-zero length must be specified with a uintbe initialiser.")
+ self._bitstore = bitstore_helpers.int2bitstore(uintbe, length, False)
- def _getuintbe(self) ->int:
+ def _getuintbe(self) -> int:
"""Return data as a big-endian two's complement unsigned int."""
- pass
+ if len(self) % 8:
+ raise bitstring.InterpretError(f"Big-endian integers must be whole-byte. Length = {len(self)} bits.")
+ return self._getuint()
- def _setintbe(self, intbe: int, length: Optional[int]=None) ->None:
+ def _setintbe(self, intbe: int, length: Optional[int] = None) -> None:
"""Set bitstring to a big-endian signed int interpretation."""
- pass
+ if length is None and hasattr(self, 'len') and len(self) != 0:
+ length = len(self)
+ if length is None or length == 0:
+ raise bitstring.CreationError("A non-zero length must be specified with a intbe initialiser.")
+ self._bitstore = bitstore_helpers.int2bitstore(intbe, length, True)
- def _getintbe(self) ->int:
+ def _getintbe(self) -> int:
"""Return data as a big-endian two's complement signed int."""
- pass
-
- def _getuintle(self) ->int:
+ if len(self) % 8:
+ raise bitstring.InterpretError(f"Big-endian integers must be whole-byte. Length = {len(self)} bits.")
+ return self._getint()
+
+ def _setuintle(self, uintle: int, length: Optional[int] = None) -> None:
+ if length is None and hasattr(self, 'len') and len(self) != 0:
+ length = len(self)
+ if length is None or length == 0:
+ raise bitstring.CreationError("A non-zero length must be specified with a uintle initialiser.")
+ self._bitstore = bitstore_helpers.intle2bitstore(uintle, length, False)
+
+ def _getuintle(self) -> int:
"""Interpret as a little-endian unsigned int."""
- pass
-
- def _getintle(self) ->int:
+ if len(self) % 8:
+ raise bitstring.InterpretError(f"Little-endian integers must be whole-byte. Length = {len(self)} bits.")
+ bs = BitStore.frombytes(self._bitstore.tobytes()[::-1])
+ return bs.slice_to_uint()
+
+ def _setintle(self, intle: int, length: Optional[int] = None) -> None:
+ if length is None and hasattr(self, 'len') and len(self) != 0:
+ length = len(self)
+ if length is None or length == 0:
+ raise bitstring.CreationError("A non-zero length must be specified with an intle initialiser.")
+ self._bitstore = bitstore_helpers.intle2bitstore(intle, length, True)
+
+ def _getintle(self) -> int:
"""Interpret as a little-endian signed int."""
- pass
-
- def _getfloatbe(self) ->float:
+ if len(self) % 8:
+ raise bitstring.InterpretError(f"Little-endian integers must be whole-byte. Length = {len(self)} bits.")
+ bs = BitStore.frombytes(self._bitstore.tobytes()[::-1])
+ return bs.slice_to_int()
+
+ def _getp4binary(self) -> float:
+ u = self._getuint()
+ return p4binary_fmt.lut_binary8_to_float[u]
+
+ def _getp3binary(self) -> float:
+ u = self._getuint()
+ return p3binary_fmt.lut_binary8_to_float[u]
+
+ def _gete4m3mxfp(self) -> float:
+ u = self._getuint()
+ return e4m3mxfp_saturate_fmt.lut_int_to_float[u]
+
+ def _gete5m2mxfp(self) -> float:
+ u = self._getuint()
+ return e5m2mxfp_saturate_fmt.lut_int_to_float[u]
+
+ def _gete3m2mxfp(self) -> float:
+ u = self._getuint()
+ return e3m2mxfp_fmt.lut_int_to_float[u]
+
+ def _gete2m3mxfp(self) -> float:
+ u = self._getuint()
+ return e2m3mxfp_fmt.lut_int_to_float[u]
+
+ def _gete2m1mxfp(self) -> float:
+ u = self._getuint()
+ return e2m1mxfp_fmt.lut_int_to_float[u]
+
+ def _gete8m0mxfp(self) -> float:
+ u = self._getuint() - 127
+ if u == 128:
+ return float('nan')
+ return 2.0 ** u
+
+ def _getmxint(self) -> float:
+ u = self._getint()
+ return float(u) * 2 ** -6
+
+ def _setfloat(self, f: float, length: Optional[int], big_endian: bool) -> None:
+ if length is None and hasattr(self, 'len') and len(self) != 0:
+ length = len(self)
+ if length is None or length not in [16, 32, 64]:
+ raise bitstring.CreationError("A length of 16, 32, or 64 must be specified with a float initialiser.")
+ self._bitstore = bitstore_helpers.float2bitstore(f, length, big_endian)
+
+ def _setfloatbe(self, f: float, length: Optional[int] = None) -> None:
+ self._setfloat(f, length, True)
+
+ def _getfloatbe(self) -> float:
"""Interpret the whole bitstring as a big-endian float."""
- pass
+ fmt = {16: '>e', 32: '>f', 64: '>d'}[len(self)]
+ return struct.unpack(fmt, self._bitstore.tobytes())[0]
- def _getfloatle(self) ->float:
+ def _setfloatle(self, f: float, length: Optional[int] = None) -> None:
+ self._setfloat(f, length, False)
+
+ def _getfloatle(self) -> float:
"""Interpret the whole bitstring as a little-endian float."""
- pass
+ fmt = {16: '<e', 32: '<f', 64: '<d'}[len(self)]
+ return struct.unpack(fmt, self._bitstore.tobytes())[0]
+
+ def _getbfloatbe(self) -> float:
+ zero_padded = self + Bits(16)
+ return zero_padded._getfloatbe()
+
+ def _setbfloatbe(self, f: Union[float, str], length: Optional[int] = None) -> None:
+ if length is not None and length != 16:
+ raise bitstring.CreationError(f"bfloats must be length 16, received a length of {length} bits.")
+ self._bitstore = bitstore_helpers.bfloat2bitstore(f, True)
- def _setue(self, i: int) ->None:
+ def _getbfloatle(self) -> float:
+ zero_padded = Bits(16) + self
+ return zero_padded._getfloatle()
+
+ def _setbfloatle(self, f: Union[float, str], length: Optional[int] = None) -> None:
+ if length is not None and length != 16:
+ raise bitstring.CreationError(f"bfloats must be length 16, received a length of {length} bits.")
+ self._bitstore = bitstore_helpers.bfloat2bitstore(f, False)
+
+ def _setue(self, i: int) -> None:
"""Initialise bitstring with unsigned exponential-Golomb code for integer i.
Raises CreationError if i < 0.
"""
- pass
+ if bitstring.options.lsb0:
+ raise bitstring.CreationError("Exp-Golomb codes cannot be used in lsb0 mode.")
+ self._bitstore = bitstore_helpers.ue2bitstore(i)
- def _readue(self, pos: int) ->Tuple[int, int]:
+ def _readue(self, pos: int) -> Tuple[int, int]:
"""Return interpretation of next bits as unsigned exponential-Golomb code.
Raises ReadError if the end of the bitstring is encountered while
reading the code.
"""
- pass
+ if bitstring.options.lsb0:
+ raise bitstring.ReadError("Exp-Golomb codes cannot be read in lsb0 mode.")
+ oldpos = pos
+ try:
+ while not self[pos]:
+ pos += 1
+ except IndexError:
+ raise bitstring.ReadError("Read off end of bitstring trying to read code.")
+ leadingzeros = pos - oldpos
+ codenum = (1 << leadingzeros) - 1
+ if leadingzeros > 0:
+ if pos + leadingzeros + 1 > len(self):
+ raise bitstring.ReadError("Read off end of bitstring trying to read code.")
+ codenum += self[pos + 1:pos + 1 + leadingzeros]._getuint()
+ pos += leadingzeros + 1
+ else:
+ assert codenum == 0
+ pos += 1
+ return codenum, pos
+
+ def _getue(self) -> Tuple[int, int]:
+ try:
+ return self._readue(0)
+ except bitstring.ReadError:
+ raise bitstring.InterpretError
+
+ def _getse(self) -> Tuple[int, int]:
+ try:
+ return self._readse(0)
+ except bitstring.ReadError:
+ raise bitstring.InterpretError
+
+ def _getuie(self) -> Tuple[int, int]:
+ try:
+ return self._readuie(0)
+ except bitstring.ReadError:
+ raise bitstring.InterpretError
- def _setse(self, i: int) ->None:
+ def _getsie(self) -> Tuple[int, int]:
+ try:
+ return self._readsie(0)
+ except bitstring.ReadError:
+ raise bitstring.InterpretError
+
+ def _setse(self, i: int) -> None:
"""Initialise bitstring with signed exponential-Golomb code for integer i."""
- pass
+ if bitstring.options.lsb0:
+ raise bitstring.CreationError("Exp-Golomb codes cannot be used in lsb0 mode.")
+ self._bitstore = bitstore_helpers.se2bitstore(i)
- def _readse(self, pos: int) ->Tuple[int, int]:
+ def _readse(self, pos: int) -> Tuple[int, int]:
"""Return interpretation of next bits as a signed exponential-Golomb code.
Advances position to after the read code.
@@ -535,30 +887,48 @@ class Bits:
reading the code.
"""
- pass
+ codenum, pos = self._readue(pos)
+ m = (codenum + 1) // 2
+ return (m, pos) if codenum % 2 else (-m, pos)
- def _setuie(self, i: int) ->None:
+ def _setuie(self, i: int) -> None:
"""Initialise bitstring with unsigned interleaved exponential-Golomb code for integer i.
Raises CreationError if i < 0.
"""
- pass
+ if bitstring.options.lsb0:
+ raise bitstring.CreationError("Exp-Golomb codes cannot be used in lsb0 mode.")
+ self._bitstore = bitstore_helpers.uie2bitstore(i)
- def _readuie(self, pos: int) ->Tuple[int, int]:
+ def _readuie(self, pos: int) -> Tuple[int, int]:
"""Return interpretation of next bits as unsigned interleaved exponential-Golomb code.
Raises ReadError if the end of the bitstring is encountered while
reading the code.
"""
- pass
-
- def _setsie(self, i: int) ->None:
+ if bitstring.options.lsb0:
+ raise bitstring.ReadError("Exp-Golomb codes cannot be read in lsb0 mode.")
+ try:
+ codenum: int = 1
+ while not self[pos]:
+ pos += 1
+ codenum <<= 1
+ codenum += self[pos]
+ pos += 1
+ pos += 1
+ except IndexError:
+ raise bitstring.ReadError("Read off end of bitstring trying to read code.")
+ return codenum - 1, pos
+
+ def _setsie(self, i: int, ) -> None:
"""Initialise bitstring with signed interleaved exponential-Golomb code for integer i."""
- pass
+ if bitstring.options.lsb0:
+ raise bitstring.CreationError("Exp-Golomb codes cannot be used in lsb0 mode.")
+ self._bitstore = bitstore_helpers.sie2bitstore(i)
- def _readsie(self, pos: int) ->Tuple[int, int]:
+ def _readsie(self, pos: int) -> Tuple[int, int]:
"""Return interpretation of next bits as a signed interleaved exponential-Golomb code.
Advances position to after the read code.
@@ -567,121 +937,219 @@ class Bits:
reading the code.
"""
- pass
+ codenum, pos = self._readuie(pos)
+ if not codenum:
+ return 0, pos
+ try:
+ return (-codenum, pos + 1) if self[pos] else (codenum, pos + 1)
+ except IndexError:
+ raise bitstring.ReadError("Read off end of bitstring trying to read code.")
+
+ def _setbool(self, value: Union[bool, str]) -> None:
+ # We deliberately don't want to have implicit conversions to bool here.
+ # If we did then it would be difficult to deal with the 'False' string.
+ if value in (1, 'True', '1'):
+ self._bitstore = BitStore('1')
+ elif value in (0, 'False', '0'):
+ self._bitstore = BitStore('0')
+ else:
+ raise bitstring.CreationError(f"Cannot initialise boolean with {value}.")
- def _setbin_safe(self, binstring: str, length: None=None) ->None:
+ def _getbool(self) -> bool:
+ return self[0]
+
+ def _getpad(self) -> None:
+ return None
+
+ def _setpad(self, value: None, length: int) -> None:
+ self._bitstore = BitStore(length)
+
+ def _setbin_safe(self, binstring: str, length: None = None) -> None:
"""Reset the bitstring to the value given in binstring."""
- pass
+ self._bitstore = bitstore_helpers.bin2bitstore(binstring)
- def _setbin_unsafe(self, binstring: str, length: None=None) ->None:
+ def _setbin_unsafe(self, binstring: str, length: None = None) -> None:
"""Same as _setbin_safe, but input isn't sanity checked. binstring mustn't start with '0b'."""
- pass
+ self._bitstore = bitstore_helpers.bin2bitstore_unsafe(binstring)
- def _getbin(self) ->str:
+ def _getbin(self) -> str:
"""Return interpretation as a binary string."""
- pass
+ return self._bitstore.slice_to_bin()
- def _setoct(self, octstring: str, length: None=None) ->None:
+ def _setoct(self, octstring: str, length: None = None) -> None:
"""Reset the bitstring to have the value given in octstring."""
- pass
+ self._bitstore = bitstore_helpers.oct2bitstore(octstring)
- def _getoct(self) ->str:
+ def _getoct(self) -> str:
"""Return interpretation as an octal string."""
- pass
+ return self._bitstore.slice_to_oct()
- def _sethex(self, hexstring: str, length: None=None) ->None:
+ def _sethex(self, hexstring: str, length: None = None) -> None:
"""Reset the bitstring to have the value given in hexstring."""
- pass
+ self._bitstore = bitstore_helpers.hex2bitstore(hexstring)
- def _gethex(self) ->str:
+ def _gethex(self) -> str:
"""Return the hexadecimal representation as a string.
Raises an InterpretError if the bitstring's length is not a multiple of 4.
"""
- pass
+ return self._bitstore.slice_to_hex()
- def _getlength(self) ->int:
+ def _getlength(self) -> int:
"""Return the length of the bitstring in bits."""
- pass
+ return len(self._bitstore)
- def _copy(self: TBits) ->TBits:
+ def _copy(self: TBits) -> TBits:
"""Create and return a new copy of the Bits (always in memory)."""
- pass
+ # Note that __copy__ may choose to return self if it's immutable. This method always makes a copy.
+ s_copy = self.__class__()
+ s_copy._bitstore = self._bitstore._copy()
+ return s_copy
- def _slice(self: TBits, start: int, end: int) ->TBits:
+ def _slice(self: TBits, start: int, end: int) -> TBits:
"""Used internally to get a slice, without error checking."""
- pass
+ bs = self.__class__()
+ bs._bitstore = self._bitstore.getslice(start, end)
+ return bs
- def _absolute_slice(self: TBits, start: int, end: int) ->TBits:
+ def _absolute_slice(self: TBits, start: int, end: int) -> TBits:
"""Used internally to get a slice, without error checking.
Uses MSB0 bit numbering even if LSB0 is set."""
- pass
+ if end == start:
+ return self.__class__()
+ assert start < end, f"start={start}, end={end}"
+ bs = self.__class__()
+ bs._bitstore = self._bitstore.getslice_msb0(start, end)
+ return bs
- def _readtoken(self, name: str, pos: int, length: Optional[int]) ->Tuple[
- Union[float, int, str, None, Bits], int]:
+ def _readtoken(self, name: str, pos: int, length: Optional[int]) -> Tuple[Union[float, int, str, None, Bits], int]:
"""Reads a token from the bitstring and returns the result."""
- pass
+ dtype = dtype_register.get_dtype(name, length)
+ if dtype.bitlength is not None and dtype.bitlength > len(self) - pos:
+ raise bitstring.ReadError("Reading off the end of the data. "
+ f"Tried to read {dtype.bitlength} bits when only {len(self) - pos} available.")
+ try:
+ val = dtype.read_fn(self, pos)
+ if isinstance(val, tuple):
+ return val
+ else:
+ assert length is not None
+ return val, pos + dtype.bitlength
+ except KeyError:
+ raise ValueError(f"Can't parse token {name}:{length}")
- def _addright(self, bs: Bits, /) ->None:
+ def _addright(self, bs: Bits, /) -> None:
"""Add a bitstring to the RHS of the current bitstring."""
- pass
+ self._bitstore += bs._bitstore
- def _addleft(self, bs: Bits, /) ->None:
+ def _addleft(self, bs: Bits, /) -> None:
"""Prepend a bitstring to the current bitstring."""
- pass
+ if bs._bitstore.immutable:
+ self._bitstore = bs._bitstore._copy() + self._bitstore
+ else:
+ self._bitstore = bs._bitstore + self._bitstore
- def _truncateleft(self: TBits, bits: int, /) ->TBits:
+ def _truncateleft(self: TBits, bits: int, /) -> TBits:
"""Truncate bits from the start of the bitstring. Return the truncated bits."""
- pass
-
- def _truncateright(self: TBits, bits: int, /) ->TBits:
+ assert 0 <= bits <= len(self)
+ if bits == 0:
+ return self.__class__()
+ truncated_bits = self._absolute_slice(0, bits)
+ if bits == len(self):
+ self._clear()
+ return truncated_bits
+ self._bitstore = self._bitstore.getslice_msb0(bits, None)
+ return truncated_bits
+
+ def _truncateright(self: TBits, bits: int, /) -> TBits:
"""Truncate bits from the end of the bitstring. Return the truncated bits."""
- pass
-
- def _insert(self, bs: Bits, pos: int, /) ->None:
+ assert 0 <= bits <= len(self)
+ if bits == 0:
+ return self.__class__()
+ truncated_bits = self._absolute_slice(len(self) - bits, len(self))
+ if bits == len(self):
+ self._clear()
+ return truncated_bits
+ self._bitstore = self._bitstore.getslice_msb0(None, -bits)
+ return truncated_bits
+
+ def _insert(self, bs: Bits, pos: int, /) -> None:
"""Insert bs at pos."""
- pass
+ assert 0 <= pos <= len(self)
+ self._bitstore[pos: pos] = bs._bitstore
+ return
- def _overwrite(self, bs: Bits, pos: int, /) ->None:
+ def _overwrite(self, bs: Bits, pos: int, /) -> None:
"""Overwrite with bs at pos."""
- pass
+ assert 0 <= pos <= len(self)
+ if bs is self:
+ # Just overwriting with self, so do nothing.
+ assert pos == 0
+ return
+ self._bitstore[pos: pos + len(bs)] = bs._bitstore
- def _delete(self, bits: int, pos: int, /) ->None:
+ def _delete(self, bits: int, pos: int, /) -> None:
"""Delete bits at pos."""
- pass
+ assert 0 <= pos <= len(self)
+ assert pos + bits <= len(self), f"pos={pos}, bits={bits}, len={len(self)}"
+ del self._bitstore[pos: pos + bits]
+ return
- def _reversebytes(self, start: int, end: int) ->None:
+ def _reversebytes(self, start: int, end: int) -> None:
"""Reverse bytes in-place."""
- pass
+ assert (end - start) % 8 == 0
+ self._bitstore[start:end] = BitStore.frombytes(self._bitstore.getslice(start, end).tobytes()[::-1])
- def _invert(self, pos: int, /) ->None:
+ def _invert(self, pos: int, /) -> None:
"""Flip bit at pos 1<->0."""
- pass
+ assert 0 <= pos < len(self)
+ self._bitstore.invert(pos)
- def _invert_all(self) ->None:
+ def _invert_all(self) -> None:
"""Invert every bit."""
- pass
+ self._bitstore.invert()
- def _ilshift(self: TBits, n: int, /) ->TBits:
+ def _ilshift(self: TBits, n: int, /) -> TBits:
"""Shift bits by n to the left in place. Return self."""
- pass
+ assert 0 < n <= len(self)
+ self._addright(Bits(n))
+ self._truncateleft(n)
+ return self
- def _irshift(self: TBits, n: int, /) ->TBits:
+ def _irshift(self: TBits, n: int, /) -> TBits:
"""Shift bits by n to the right in place. Return self."""
- pass
+ assert 0 < n <= len(self)
+ self._addleft(Bits(n))
+ self._truncateright(n)
+ return self
- def _imul(self: TBits, n: int, /) ->TBits:
+ def _imul(self: TBits, n: int, /) -> TBits:
"""Concatenate n copies of self in place. Return self."""
- pass
+ assert n >= 0
+ if n == 0:
+ self._clear()
+ else:
+ m = 1
+ old_len = len(self)
+ while m * 2 < n:
+ self._addright(self)
+ m *= 2
+ self._addright(self[0:(n - m) * old_len])
+ return self
- def _validate_slice(self, start: Optional[int], end: Optional[int]
- ) ->Tuple[int, int]:
+ def _getbits(self: TBits):
+ return self._copy()
+
+ def _validate_slice(self, start: Optional[int], end: Optional[int]) -> Tuple[int, int]:
"""Validate start and end and return them as positive bit positions."""
- pass
+ start = 0 if start is None else (start + len(self) if start < 0 else start)
+ end = len(self) if end is None else (end + len(self) if end < 0 else end)
+ if not 0 <= start <= end <= len(self):
+ raise ValueError(f"Invalid slice positions for bitstring length {len(self)}: start={start}, end={end}.")
+ return start, end
- def unpack(self, fmt: Union[str, List[Union[str, int]]], **kwargs) ->List[
- Union[int, float, str, Bits, bool, bytes, None]]:
+ def unpack(self, fmt: Union[str, List[Union[str, int]]], **kwargs) -> List[Union[int, float, str, Bits, bool, bytes, None]]:
"""Interpret the whole bitstring using fmt and return list.
fmt -- A single string or a list of strings with comma separated tokens
@@ -696,11 +1164,69 @@ class Bits:
See the docstring for 'read' for token examples.
"""
- pass
+ return self._readlist(fmt, 0, **kwargs)[0]
+
+ def _readlist(self, fmt: Union[str, List[Union[str, int, Dtype]]], pos: int, **kwargs) \
+ -> Tuple[List[Union[int, float, str, Bits, bool, bytes, None]], int]:
+ if isinstance(fmt, str):
+ fmt = [fmt]
+ # Convert to a flat list of Dtypes
+ dtype_list = []
+ for f_item in fmt:
+ if isinstance(f_item, numbers.Integral):
+ dtype_list.append(Dtype('bits', f_item))
+ elif isinstance(f_item, Dtype):
+ dtype_list.append(f_item)
+ else:
+ token_list = utils.preprocess_tokens(f_item)
+ for t in token_list:
+ try:
+ name, length = utils.parse_name_length_token(t, **kwargs)
+ except ValueError:
+ dtype_list.append(Dtype('bits', int(t)))
+ else:
+ dtype_list.append(Dtype(name, length))
+ return self._read_dtype_list(dtype_list, pos)
+
+ def _read_dtype_list(self, dtypes: List[Dtype], pos: int) -> Tuple[List[Union[int, float, str, Bits, bool, bytes, None]], int]:
+ has_stretchy_token = False
+ bits_after_stretchy_token = 0
+ for dtype in dtypes:
+ stretchy = dtype.bitlength is None and not dtype.variable_length
+ if stretchy:
+ if has_stretchy_token:
+ raise bitstring.Error("It's not possible to have more than one 'filler' token.")
+ has_stretchy_token = True
+ elif has_stretchy_token:
+ if dtype.variable_length:
+ raise bitstring.Error(f"It's not possible to parse a variable length token '{dtype}' after a 'filler' token.")
+ bits_after_stretchy_token += dtype.bitlength
+
+ # We should have precisely zero or one stretchy token
+ vals = []
+ for dtype in dtypes:
+ stretchy = dtype.bitlength is None and not dtype.variable_length
+ if stretchy:
+ bits_remaining = len(self) - pos
+ # Set length to the remaining bits
+ bitlength = max(bits_remaining - bits_after_stretchy_token, 0)
+ items, remainder = divmod(bitlength, dtype.bits_per_item)
+ if remainder != 0:
+ raise ValueError(
+ f"The '{dtype.name}' type must have a bit length that is a multiple of {dtype.bits_per_item}"
+ f" so cannot be created from the {bitlength} bits that are available for this stretchy token.")
+ dtype = Dtype(dtype.name, items)
+ if dtype.bitlength is not None:
+ val = dtype.read_fn(self, pos)
+ pos += dtype.bitlength
+ else:
+ val, pos = dtype.read_fn(self, pos)
+ if val is not None: # Don't append pad tokens
+ vals.append(val)
+ return vals, pos
- def find(self, bs: BitsType, /, start: Optional[int]=None, end:
- Optional[int]=None, bytealigned: Optional[bool]=None) ->Union[Tuple
- [int], Tuple[()]]:
+ def find(self, bs: BitsType, /, start: Optional[int] = None, end: Optional[int] = None,
+ bytealigned: Optional[bool] = None) -> Union[Tuple[int], Tuple[()]]:
"""Find first occurrence of substring bs.
Returns a single item tuple with the bit position if found, or an
@@ -721,16 +1247,35 @@ class Bits:
(6,)
"""
- pass
+ bs = Bits._create_from_bitstype(bs)
+ if len(bs) == 0:
+ raise ValueError("Cannot find an empty bitstring.")
+ start, end = self._validate_slice(start, end)
+ ba = bitstring.options.bytealigned if bytealigned is None else bytealigned
+ p = self._find(bs, start, end, ba)
+ return p
+
+ def _find_lsb0(self, bs: Bits, start: int, end: int, bytealigned: bool) -> Union[Tuple[int], Tuple[()]]:
+ # A forward find in lsb0 is very like a reverse find in msb0.
+ assert start <= end
+ assert bitstring.options.lsb0
+
+ new_slice = bitstring.bitstore.offset_slice_indices_lsb0(slice(start, end, None), len(self))
+ msb0_start, msb0_end = self._validate_slice(new_slice.start, new_slice.stop)
+ p = self._rfind_msb0(bs, msb0_start, msb0_end, bytealigned)
+
+ if p:
+ return (len(self) - p[0] - len(bs),)
+ else:
+ return ()
- def _find_msb0(self, bs: Bits, start: int, end: int, bytealigned: bool
- ) ->Union[Tuple[int], Tuple[()]]:
+ def _find_msb0(self, bs: Bits, start: int, end: int, bytealigned: bool) -> Union[Tuple[int], Tuple[()]]:
"""Find first occurrence of a binary string."""
- pass
+ p = self._bitstore.find(bs._bitstore, start, end, bytealigned)
+ return () if p == -1 else (p,)
- def findall(self, bs: BitsType, start: Optional[int]=None, end:
- Optional[int]=None, count: Optional[int]=None, bytealigned:
- Optional[bool]=None) ->Iterable[int]:
+ def findall(self, bs: BitsType, start: Optional[int] = None, end: Optional[int] = None, count: Optional[int] = None,
+ bytealigned: Optional[bool] = None) -> Iterable[int]:
"""Find all occurrences of bs. Return generator of bit positions.
bs -- The bitstring to find.
@@ -747,11 +1292,57 @@ class Bits:
Note that all occurrences of bs are found, even if they overlap.
"""
- pass
-
- def rfind(self, bs: BitsType, /, start: Optional[int]=None, end:
- Optional[int]=None, bytealigned: Optional[bool]=None) ->Union[Tuple
- [int], Tuple[()]]:
+ if count is not None and count < 0:
+ raise ValueError("In findall, count must be >= 0.")
+ bs = Bits._create_from_bitstype(bs)
+ start, end = self._validate_slice(start, end)
+ ba = bitstring.options.bytealigned if bytealigned is None else bytealigned
+ return self._findall(bs, start, end, count, ba)
+
+ def _findall_msb0(self, bs: Bits, start: int, end: int, count: Optional[int],
+ bytealigned: bool) -> Iterable[int]:
+ c = 0
+ for i in self._bitstore.findall_msb0(bs._bitstore, start, end, bytealigned):
+ if count is not None and c >= count:
+ return
+ c += 1
+ yield i
+ return
+
+ def _findall_lsb0(self, bs: Bits, start: int, end: int, count: Optional[int],
+ bytealigned: bool) -> Iterable[int]:
+ assert start <= end
+ assert bitstring.options.lsb0
+
+ new_slice = bitstring.bitstore.offset_slice_indices_lsb0(slice(start, end, None), len(self))
+ msb0_start, msb0_end = self._validate_slice(new_slice.start, new_slice.stop)
+
+ # Search chunks starting near the end and then moving back.
+ c = 0
+ increment = max(8192, len(bs) * 80)
+ buffersize = min(increment + len(bs), msb0_end - msb0_start)
+ pos = max(msb0_start, msb0_end - buffersize)
+ while True:
+ found = list(self._findall_msb0(bs, start=pos, end=pos + buffersize, count=None, bytealigned=False))
+ if not found:
+ if pos == msb0_start:
+ return
+ pos = max(msb0_start, pos - increment)
+ continue
+ while found:
+ if count is not None and c >= count:
+ return
+ c += 1
+ lsb0_pos = len(self) - found.pop() - len(bs)
+ if not bytealigned or lsb0_pos % 8 == 0:
+ yield lsb0_pos
+
+ pos = max(msb0_start, pos - increment)
+ if pos == msb0_start:
+ return
+
+ def rfind(self, bs: BitsType, /, start: Optional[int] = None, end: Optional[int] = None,
+ bytealigned: Optional[bool] = None) -> Union[Tuple[int], Tuple[()]]:
"""Find final occurrence of substring bs.
Returns a single item tuple with the bit position if found, or an
@@ -769,15 +1360,34 @@ class Bits:
if end < start.
"""
- pass
-
- def _rfind_msb0(self, bs: Bits, start: int, end: int, bytealigned: bool
- ) ->Union[Tuple[int], Tuple[()]]:
+ bs = Bits._create_from_bitstype(bs)
+ start, end = self._validate_slice(start, end)
+ ba = bitstring.options.bytealigned if bytealigned is None else bytealigned
+ if len(bs) == 0:
+ raise ValueError("Cannot find an empty bitstring.")
+ p = self._rfind(bs, start, end, ba)
+ return p
+
+ def _rfind_msb0(self, bs: Bits, start: int, end: int, bytealigned: bool) -> Union[Tuple[int], Tuple[()]]:
"""Find final occurrence of a binary string."""
- pass
+ p = self._bitstore.rfind(bs._bitstore, start, end, bytealigned)
+ return () if p == -1 else (p,)
+
+ def _rfind_lsb0(self, bs: Bits, start: int, end: int, bytealigned: bool) -> Union[Tuple[int], Tuple[()]]:
+ # A reverse find in lsb0 is very like a forward find in msb0.
+ assert start <= end
+ assert bitstring.options.lsb0
+ new_slice = bitstring.bitstore.offset_slice_indices_lsb0(slice(start, end, None), len(self))
+ msb0_start, msb0_end = self._validate_slice(new_slice.start, new_slice.stop)
+
+ p = self._find_msb0(bs, msb0_start, msb0_end, bytealigned)
+ if p:
+ return (len(self) - p[0] - len(bs),)
+ else:
+ return ()
- def cut(self, bits: int, start: Optional[int]=None, end: Optional[int]=
- None, count: Optional[int]=None) ->Iterator[Bits]:
+ def cut(self, bits: int, start: Optional[int] = None, end: Optional[int] = None,
+ count: Optional[int] = None) -> Iterator[Bits]:
"""Return bitstring generator by cutting into bits sized chunks.
bits -- The size in bits of the bitstring chunks to generate.
@@ -788,11 +1398,25 @@ class Bits:
Default is to cut as many times as possible.
"""
- pass
-
- def split(self, delimiter: BitsType, start: Optional[int]=None, end:
- Optional[int]=None, count: Optional[int]=None, bytealigned:
- Optional[bool]=None) ->Iterable[Bits]:
+ start_, end_ = self._validate_slice(start, end)
+ if count is not None and count < 0:
+ raise ValueError("Cannot cut - count must be >= 0.")
+ if bits <= 0:
+ raise ValueError("Cannot cut - bits must be >= 0.")
+ c = 0
+ while count is None or c < count:
+ c += 1
+ nextchunk = self._slice(start_, min(start_ + bits, end_))
+ if len(nextchunk) == 0:
+ return
+ yield nextchunk
+ if len(nextchunk) != bits:
+ return
+ start_ += bits
+ return
+
+ def split(self, delimiter: BitsType, start: Optional[int] = None, end: Optional[int] = None,
+ count: Optional[int] = None, bytealigned: Optional[bool] = None) -> Iterable[Bits]:
"""Return bitstring generator by splitting using a delimiter.
The first item returned is the initial bitstring before the delimiter,
@@ -809,38 +1433,89 @@ class Bits:
Raises ValueError if the delimiter is empty.
"""
- pass
-
- def join(self: TBits, sequence: Iterable[Any]) ->TBits:
+ delimiter = Bits._create_from_bitstype(delimiter)
+ if len(delimiter) == 0:
+ raise ValueError("split delimiter cannot be empty.")
+ start, end = self._validate_slice(start, end)
+ bytealigned_: bool = bitstring.options.bytealigned if bytealigned is None else bytealigned
+ if count is not None and count < 0:
+ raise ValueError("Cannot split - count must be >= 0.")
+ if count == 0:
+ return
+ f = functools.partial(self._find_msb0, bs=delimiter, bytealigned=bytealigned_)
+ found = f(start=start, end=end)
+ if not found:
+ # Initial bits are the whole bitstring being searched
+ yield self._slice(start, end)
+ return
+ # yield the bytes before the first occurrence of the delimiter, even if empty
+ yield self._slice(start, found[0])
+ startpos = pos = found[0]
+ c = 1
+ while count is None or c < count:
+ pos += len(delimiter)
+ found = f(start=pos, end=end)
+ if not found:
+ # No more occurrences, so return the rest of the bitstring
+ yield self._slice(startpos, end)
+ return
+ c += 1
+ yield self._slice(startpos, found[0])
+ startpos = pos = found[0]
+ # Have generated count bitstrings, so time to quit.
+ return
+
+ def join(self: TBits, sequence: Iterable[Any]) -> TBits:
"""Return concatenation of bitstrings joined by self.
sequence -- A sequence of bitstrings.
"""
- pass
-
- def tobytes(self) ->bytes:
+ s = self.__class__()
+ if len(self) == 0:
+ # Optimised version that doesn't need to add self between every item
+ for item in sequence:
+ s._addright(Bits._create_from_bitstype(item))
+ return s
+ else:
+ sequence_iter = iter(sequence)
+ try:
+ s._addright(Bits._create_from_bitstype(next(sequence_iter)))
+ except StopIteration:
+ return s
+ for item in sequence_iter:
+ s._addright(self)
+ s._addright(Bits._create_from_bitstype(item))
+ return s
+
+ def tobytes(self) -> bytes:
"""Return the bitstring as bytes, padding with zero bits if needed.
Up to seven zero bits will be added at the end to byte align.
"""
- pass
+ return self._bitstore.tobytes()
- def tobitarray(self) ->bitarray.bitarray:
+ def tobitarray(self) -> bitarray.bitarray:
"""Convert the bitstring to a bitarray object."""
- pass
+ if self._bitstore.modified_length is not None:
+ # Removes the offset and truncates to length
+ return self._bitstore.getslice(0, len(self))._bitarray
+ else:
+ return self._bitstore._bitarray
- def tofile(self, f: BinaryIO) ->None:
+ def tofile(self, f: BinaryIO) -> None:
"""Write the bitstring to a file object, padding with zero bits if needed.
Up to seven zero bits will be added at the end to byte align.
"""
- pass
+ # If the bitstring is file based then we don't want to read it all in to memory first.
+ chunk_size = 8 * 100 * 1024 * 1024 # 100 MiB
+ for chunk in self.cut(chunk_size):
+ f.write(chunk.tobytes())
- def startswith(self, prefix: BitsType, start: Optional[int]=None, end:
- Optional[int]=None) ->bool:
+ def startswith(self, prefix: BitsType, start: Optional[int] = None, end: Optional[int] = None) -> bool:
"""Return whether the current bitstring starts with prefix.
prefix -- The bitstring to search for.
@@ -848,10 +1523,11 @@ class Bits:
end -- The bit position to end at. Defaults to len(self).
"""
- pass
+ prefix = self._create_from_bitstype(prefix)
+ start, end = self._validate_slice(start, end)
+ return self._slice(start, start + len(prefix)) == prefix if end >= start + len(prefix) else False
- def endswith(self, suffix: BitsType, start: Optional[int]=None, end:
- Optional[int]=None) ->bool:
+ def endswith(self, suffix: BitsType, start: Optional[int] = None, end: Optional[int] = None) -> bool:
"""Return whether the current bitstring ends with suffix.
suffix -- The bitstring to search for.
@@ -859,9 +1535,11 @@ class Bits:
end -- The bit position to end at. Defaults to len(self).
"""
- pass
+ suffix = self._create_from_bitstype(suffix)
+ start, end = self._validate_slice(start, end)
+ return self._slice(end - len(suffix), end) == suffix if start + len(suffix) <= end else False
- def all(self, value: Any, pos: Optional[Iterable[int]]=None) ->bool:
+ def all(self, value: Any, pos: Optional[Iterable[int]] = None) -> bool:
"""Return True if one or many bits are all set to bool(value).
value -- If value is True then checks for bits set to 1, otherwise
@@ -870,9 +1548,15 @@ class Bits:
the same way as slice indices. Defaults to the whole bitstring.
"""
- pass
-
- def any(self, value: Any, pos: Optional[Iterable[int]]=None) ->bool:
+ value = 1 if bool(value) else 0
+ if pos is None:
+ return self._bitstore.all_set() if value else not self._bitstore.any_set()
+ for p in pos:
+ if self._bitstore.getindex(p) != value:
+ return False
+ return True
+
+ def any(self, value: Any, pos: Optional[Iterable[int]] = None) -> bool:
"""Return True if any of one or many bits are set to bool(value).
value -- If value is True then checks for bits set to 1, otherwise
@@ -881,9 +1565,15 @@ class Bits:
the same way as slice indices. Defaults to the whole bitstring.
"""
- pass
-
- def count(self, value: Any) ->int:
+ value = 1 if bool(value) else 0
+ if pos is None:
+ return self._bitstore.any_set() if value else not self._bitstore.all_set()
+ for p in pos:
+ if self._bitstore.getindex(p) == value:
+ return True
+ return False
+
+ def count(self, value: Any) -> int:
"""Return count of total number of either zero or one bits.
value -- If bool(value) is True then bits set to 1 are counted, otherwise bits set
@@ -893,26 +1583,157 @@ class Bits:
7
"""
- pass
+ # count the number of 1s (from which it's easy to work out the 0s).
+ count = self._bitstore.count(1)
+ return count if value else len(self) - count
+
+ @staticmethod
+ def _format_bits(bits: Bits, bits_per_group: int, sep: str, dtype: Dtype,
+ colour_start: str, colour_end: str, width: Optional[int]=None) -> Tuple[str, int]:
+ get_fn = dtype.get_fn
+ if dtype.name == 'bytes': # Special case for bytes to print one character each.
+ get_fn = Bits._getbytes_printable
+ if dtype.name == 'bool': # Special case for bool to print '1' or '0' instead of `True` or `False`.
+ get_fn = dtype_register.get_dtype('uint', bits_per_group).get_fn
+ if bits_per_group == 0:
+ x = str(get_fn(bits))
+ else:
+ # Left-align for fixed width types when msb0, otherwise right-align.
+ align = '<' if dtype.name in ['bin', 'oct', 'hex', 'bits', 'bytes'] and not bitstring.options.lsb0 else '>'
+ chars_per_group = 0
+ if dtype_register[dtype.name].bitlength2chars_fn is not None:
+ chars_per_group = dtype_register[dtype.name].bitlength2chars_fn(bits_per_group)
+ x = sep.join(f"{str(get_fn(b)): {align}{chars_per_group}}" for b in bits.cut(bits_per_group))
+
+ chars_used = len(x)
+ padding_spaces = 0 if width is None else max(width - len(x), 0)
+ x = colour_start + x + colour_end
+ # Pad final line with spaces to align it
+ if bitstring.options.lsb0:
+ x = ' ' * padding_spaces + x
+ else:
+ x += ' ' * padding_spaces
+ return x, chars_used
@staticmethod
def _chars_per_group(bits_per_group: int, fmt: Optional[str]):
"""How many characters are needed to represent a number of bits with a given format."""
- pass
+ if fmt is None or dtype_register[fmt].bitlength2chars_fn is None:
+ return 0
+ return dtype_register[fmt].bitlength2chars_fn(bits_per_group)
@staticmethod
def _bits_per_char(fmt: str):
"""How many bits are represented by each character of a given format."""
- pass
+ if fmt not in ['bin', 'oct', 'hex', 'bytes']:
+ raise ValueError
+ return 24 // dtype_register[fmt].bitlength2chars_fn(24)
- def _pp(self, dtype1: Dtype, dtype2: Optional[Dtype], bits_per_group:
- int, width: int, sep: str, format_sep: str, show_offset: bool,
- stream: TextIO, lsb0: bool, offset_factor: int) ->None:
+ def _pp(self, dtype1: Dtype, dtype2: Optional[Dtype], bits_per_group: int, width: int, sep: str, format_sep: str,
+ show_offset: bool, stream: TextIO, lsb0: bool, offset_factor: int) -> None:
"""Internal pretty print method."""
- pass
+ colour = Colour(not bitstring.options.no_color)
+ name1 = dtype1.name
+ name2 = dtype2.name if dtype2 is not None else None
+ if dtype1.variable_length:
+ raise ValueError(f"Can't use Dtype '{dtype1}' in pp() as it has a variable length.")
+ if dtype2 is not None and dtype2.variable_length:
+ raise ValueError(f"Can't use Dtype '{dtype2}' in pp() as it has a variable length.")
+ offset_width = 0
+ offset_sep = ' :' if lsb0 else ': '
+ if show_offset:
+ # This could be 1 too large in some circumstances. Slightly recurrent logic needed to fix it...
+ offset_width = len(str(len(self))) + len(offset_sep)
+ if bits_per_group > 0:
+ group_chars1 = Bits._chars_per_group(bits_per_group, name1)
+ group_chars2 = Bits._chars_per_group(bits_per_group, name2)
+ # The number of characters that get added when we add an extra group (after the first one)
+ total_group_chars = group_chars1 + group_chars2 + len(sep) + len(sep) * bool(group_chars2)
+ width_excluding_offset_and_final_group = width - offset_width - group_chars1 - group_chars2 - len(
+ format_sep) * bool(group_chars2)
+ width_excluding_offset_and_final_group = max(width_excluding_offset_and_final_group, 0)
+ groups_per_line = 1 + width_excluding_offset_and_final_group // total_group_chars
+ max_bits_per_line = groups_per_line * bits_per_group # Number of bits represented on each line
+ else:
+ assert bits_per_group == 0 # Don't divide into groups
+ width_available = width - offset_width - len(format_sep) * (name2 is not None)
+ width_available = max(width_available, 1)
+ if name2 is None:
+ max_bits_per_line = width_available * Bits._bits_per_char(name1)
+ else:
+ chars_per_24_bits = dtype_register[name1].bitlength2chars_fn(24) + dtype_register[name2].bitlength2chars_fn(24)
+ max_bits_per_line = 24 * (width_available // chars_per_24_bits)
+ if max_bits_per_line == 0:
+ max_bits_per_line = 24 # We can't fit into the width asked for. Show something small.
+ assert max_bits_per_line > 0
+
+ bitpos = 0
+ first_fb_width = second_fb_width = None
+ for bits in self.cut(max_bits_per_line):
+ offset_str = ''
+ if show_offset:
+ offset = bitpos // offset_factor
+ bitpos += len(bits)
+ if bitstring.options.lsb0:
+ offset_str = colour.green + offset_sep + f'{offset: <{offset_width - len(offset_sep)}}' + colour.off
+ else:
+ offset_str = colour.green + f'{offset: >{offset_width - len(offset_sep)}}' + offset_sep + colour.off
+
+ fb1, chars_used = Bits._format_bits(bits, bits_per_group, sep, dtype1, colour.purple, colour.off, first_fb_width)
+ if first_fb_width is None:
+ first_fb_width = chars_used
+
+ fb2 = ''
+ if dtype2 is not None:
+ fb2, chars_used = Bits._format_bits(bits, bits_per_group, sep, dtype2, colour.blue, colour.off, second_fb_width)
+ if second_fb_width is None:
+ second_fb_width = chars_used
+ fb2 = format_sep + fb2
+
+ if bitstring.options.lsb0 is True:
+ line_fmt = fb1 + fb2 + offset_str + '\n'
+ else:
+ line_fmt = offset_str + fb1 + fb2 + '\n'
+ stream.write(line_fmt)
+ return
- def pp(self, fmt: Optional[str]=None, width: int=120, sep: str=' ',
- show_offset: bool=True, stream: TextIO=sys.stdout) ->None:
+ @staticmethod
+ def _process_pp_tokens(token_list, fmt):
+ if len(token_list) not in [1, 2]:
+ raise ValueError(
+ f"Only one or two tokens can be used in an pp() format - '{fmt}' has {len(token_list)} tokens.")
+ has_length_in_fmt = True
+ name1, length1 = utils.parse_name_length_token(token_list[0])
+ dtype1 = Dtype(name1, length1)
+ bits_per_group = dtype1.bitlength
+ dtype2 = None
+
+ if len(token_list) == 2:
+ name2, length2 = utils.parse_name_length_token(token_list[1])
+ dtype2 = Dtype(name2, length2)
+ if None not in {dtype1.bitlength, dtype2.bitlength} and dtype1.bitlength != dtype2.bitlength:
+ raise ValueError(
+ f"Differing bit lengths of {dtype1.bitlength} and {dtype2.bitlength} in format string '{fmt}'.")
+ if bits_per_group is None:
+ bits_per_group = dtype2.bitlength
+
+ if bits_per_group is None:
+ has_length_in_fmt = False
+ if len(token_list) == 1:
+ bits_per_group = {'bin': 8, 'hex': 8, 'oct': 12, 'bytes': 32}.get(dtype1.name)
+ if bits_per_group is None:
+ raise ValueError(f"No length or default length available for pp() format '{fmt}'.")
+ else:
+ try:
+ bits_per_group = 2 * Bits._bits_per_char(dtype1.name) * Bits._bits_per_char(dtype2.name)
+ except ValueError:
+ raise ValueError(f"Can't find a default bitlength to use for pp() format '{fmt}'.")
+ if bits_per_group >= 24:
+ bits_per_group //= 2
+ return dtype1, dtype2, bits_per_group, has_length_in_fmt
+
+ def pp(self, fmt: Optional[str] = None, width: int = 120, sep: str = ' ',
+ show_offset: bool = True, stream: TextIO = sys.stdout) -> None:
"""Pretty print the bitstring's value.
fmt -- Printed data format. One or two of 'bin', 'oct', 'hex' or 'bytes'.
@@ -929,15 +1750,42 @@ class Bits:
>>> s.pp('b, h', sep='_', show_offset=False)
"""
- pass
-
- def copy(self: TBits) ->TBits:
+ colour = Colour(not bitstring.options.no_color)
+ if fmt is None:
+ fmt = 'bin, hex' if len(self) % 8 == 0 and len(self) >= 8 else 'bin'
+ token_list = utils.preprocess_tokens(fmt)
+ dtype1, dtype2, bits_per_group, has_length_in_fmt = Bits._process_pp_tokens(token_list, fmt)
+ trailing_bit_length = len(self) % bits_per_group if has_length_in_fmt and bits_per_group else 0
+ data = self if trailing_bit_length == 0 else self[0: -trailing_bit_length]
+ format_sep = " : " # String to insert on each line between multiple formats
+ tidy_fmt = colour.purple + str(dtype1) + colour.off
+ if dtype2 is not None:
+ tidy_fmt += ', ' + colour.blue + str(dtype2) + colour.off
+ output_stream = io.StringIO()
+ len_str = colour.green + str(len(self)) + colour.off
+ output_stream.write(f"<{self.__class__.__name__}, fmt='{tidy_fmt}', length={len_str} bits> [\n")
+ data._pp(dtype1, dtype2, bits_per_group, width, sep, format_sep, show_offset,
+ output_stream, bitstring.options.lsb0, 1)
+ output_stream.write("]")
+ if trailing_bit_length != 0:
+ output_stream.write(" + trailing_bits = " + str(self[-trailing_bit_length:]))
+ output_stream.write("\n")
+ stream.write(output_stream.getvalue())
+ return
+
+ def copy(self: TBits) -> TBits:
"""Return a copy of the bitstring."""
- pass
+ # Note that if you want a new copy (different ID), use _copy instead.
+ # The copy can return self as it's immutable.
+ return self
@classmethod
- def fromstring(cls: TBits, s: str, /) ->TBits:
+ def fromstring(cls: TBits, s: str, /) -> TBits:
"""Create a new bitstring from a formatted string."""
- pass
- len = length = property(_getlength, doc=
- 'The length of the bitstring in bits. Read only.')
+ x = super().__new__(cls)
+ x._bitstore = bitstore_helpers.str_to_bitstore(s)
+ return x
+
+ len = length = property(_getlength, doc="The length of the bitstring in bits. Read only.")
+
+
diff --git a/bitstring/bitstore.py b/bitstring/bitstore.py
index d01a2d3..cdb199f 100644
--- a/bitstring/bitstore.py
+++ b/bitstring/bitstore.py
@@ -1,63 +1,272 @@
from __future__ import annotations
+
import bitarray
from bitstring.exceptions import CreationError
from typing import Union, Iterable, Optional, overload, Iterator, Any
+def offset_slice_indices_lsb0(key: slice, length: int) -> slice:
+ # First convert slice to all integers
+ # Length already should take account of the offset
+ start, stop, step = key.indices(length)
+ new_start = length - stop
+ new_stop = length - start
+ # For negative step we sometimes get a negative stop, which can't be used correctly in a new slice
+ return slice(new_start, None if new_stop < 0 else new_stop, step)
+
+
+def offset_start_stop_lsb0(start: Optional[int], stop: Optional[int], length: int) -> tuple[int, int]:
+ # First convert slice to all integers
+ # Length already should take account of the offset
+ start, stop, _ = slice(start, stop, None).indices(length)
+ new_start = length - stop
+ new_stop = length - start
+ return new_start, new_stop
+
+
class BitStore:
"""A light wrapper around bitarray that does the LSB0 stuff"""
- __slots__ = '_bitarray', 'modified_length', 'immutable'
- def __init__(self, initializer: Union[int, bitarray.bitarray, str, None
- ]=None, immutable: bool=False) ->None:
+ __slots__ = ('_bitarray', 'modified_length', 'immutable')
+
+ def __init__(self, initializer: Union[int, bitarray.bitarray, str, None] = None,
+ immutable: bool = False) -> None:
self._bitarray = bitarray.bitarray(initializer)
self.immutable = immutable
self.modified_length = None
- def __iadd__(self, other: BitStore, /) ->BitStore:
+ @classmethod
+ def frombytes(cls, b: Union[bytes, bytearray, memoryview], /) -> BitStore:
+ x = super().__new__(cls)
+ x._bitarray = bitarray.bitarray()
+ x._bitarray.frombytes(b)
+ x.immutable = False
+ x.modified_length = None
+ return x
+
+ @classmethod
+ def frombuffer(cls, buffer, /, length: Optional[int] = None) -> BitStore:
+ x = super().__new__(cls)
+ x._bitarray = bitarray.bitarray(buffer=buffer)
+ x.immutable = True
+ x.modified_length = length
+ # Here 'modified' means it shouldn't be changed further, so setting, deleting etc. are disallowed.
+ if x.modified_length is not None:
+ if x.modified_length < 0:
+ raise CreationError("Can't create bitstring with a negative length.")
+ if x.modified_length > len(x._bitarray):
+ raise CreationError(
+ f"Can't create bitstring with a length of {x.modified_length} from {len(x._bitarray)} bits of data.")
+ return x
+
+ def setall(self, value: int, /) -> None:
+ self._bitarray.setall(value)
+
+ def tobytes(self) -> bytes:
+ if self.modified_length is not None:
+ return self._bitarray[:self.modified_length].tobytes()
+ return self._bitarray.tobytes()
+
+ def slice_to_uint(self, start: Optional[int] = None, end: Optional[int] = None) -> int:
+ return bitarray.util.ba2int(self.getslice(start, end)._bitarray, signed=False)
+
+ def slice_to_int(self, start: Optional[int] = None, end: Optional[int] = None) -> int:
+ return bitarray.util.ba2int(self.getslice(start, end)._bitarray, signed=True)
+
+ def slice_to_hex(self, start: Optional[int] = None, end: Optional[int] = None) -> str:
+ return bitarray.util.ba2hex(self.getslice(start, end)._bitarray)
+
+ def slice_to_bin(self, start: Optional[int] = None, end: Optional[int] = None) -> str:
+ return self.getslice(start, end)._bitarray.to01()
+
+ def slice_to_oct(self, start: Optional[int] = None, end: Optional[int] = None) -> str:
+ return bitarray.util.ba2base(8, self.getslice(start, end)._bitarray)
+
+ def __iadd__(self, other: BitStore, /) -> BitStore:
self._bitarray += other._bitarray
return self
- def __add__(self, other: BitStore, /) ->BitStore:
+ def __add__(self, other: BitStore, /) -> BitStore:
bs = self._copy()
bs += other
return bs
- def __eq__(self, other: Any, /) ->bool:
+ def __eq__(self, other: Any, /) -> bool:
return self._bitarray == other._bitarray
- def __and__(self, other: BitStore, /) ->BitStore:
+ def __and__(self, other: BitStore, /) -> BitStore:
return BitStore(self._bitarray & other._bitarray)
- def __or__(self, other: BitStore, /) ->BitStore:
+ def __or__(self, other: BitStore, /) -> BitStore:
return BitStore(self._bitarray | other._bitarray)
- def __xor__(self, other: BitStore, /) ->BitStore:
+ def __xor__(self, other: BitStore, /) -> BitStore:
return BitStore(self._bitarray ^ other._bitarray)
- def __iand__(self, other: BitStore, /) ->BitStore:
+ def __iand__(self, other: BitStore, /) -> BitStore:
self._bitarray &= other._bitarray
return self
- def __ior__(self, other: BitStore, /) ->BitStore:
+ def __ior__(self, other: BitStore, /) -> BitStore:
self._bitarray |= other._bitarray
return self
- def __ixor__(self, other: BitStore, /) ->BitStore:
+ def __ixor__(self, other: BitStore, /) -> BitStore:
self._bitarray ^= other._bitarray
return self
- def __iter__(self) ->Iterable[bool]:
+ def find(self, bs: BitStore, start: int, end: int, bytealigned: bool = False) -> int:
+ if not bytealigned:
+ return self._bitarray.find(bs._bitarray, start, end)
+ try:
+ return next(self.findall_msb0(bs, start, end, bytealigned))
+ except StopIteration:
+ return -1
+
+ def rfind(self, bs: BitStore, start: int, end: int, bytealigned: bool = False):
+ if not bytealigned:
+ return self._bitarray.find(bs._bitarray, start, end, right=True)
+ try:
+ return next(self.rfindall_msb0(bs, start, end, bytealigned))
+ except StopIteration:
+ return -1
+
+ def findall_msb0(self, bs: BitStore, start: int, end: int, bytealigned: bool = False) -> Iterator[int]:
+ if bytealigned is True and len(bs) % 8 == 0:
+ # Special case, looking for whole bytes on whole byte boundaries
+ bytes_ = bs.tobytes()
+ # Round up start byte to next byte, and round end byte down.
+ # We're only looking for whole bytes, so can ignore bits at either end.
+ start_byte = (start + 7) // 8
+ end_byte = end // 8
+ b = self._bitarray[start_byte * 8: end_byte * 8].tobytes()
+ byte_pos = 0
+ bytes_to_search = end_byte - start_byte
+ while byte_pos < bytes_to_search:
+ byte_pos = b.find(bytes_, byte_pos)
+ if byte_pos == -1:
+ break
+ yield (byte_pos + start_byte) * 8
+ byte_pos = byte_pos + 1
+ return
+ # General case
+ i = self._bitarray.itersearch(bs._bitarray, start, end)
+ if not bytealigned:
+ for p in i:
+ yield p
+ else:
+ for p in i:
+ if (p % 8) == 0:
+ yield p
+
+ def rfindall_msb0(self, bs: BitStore, start: int, end: int, bytealigned: bool = False) -> Iterator[int]:
+ i = self._bitarray.itersearch(bs._bitarray, start, end, right=True)
+ if not bytealigned:
+ for p in i:
+ yield p
+ else:
+ for p in i:
+ if (p % 8) == 0:
+ yield p
+
+ def count(self, value, /) -> int:
+ return self._bitarray.count(value)
+
+ def clear(self) -> None:
+ self._bitarray.clear()
+
+ def reverse(self) -> None:
+ self._bitarray.reverse()
+
+ def __iter__(self) -> Iterable[bool]:
for i in range(len(self)):
yield self.getindex(i)
- def _copy(self) ->BitStore:
+ def _copy(self) -> BitStore:
"""Always creates a copy, even if instance is immutable."""
- pass
+ return BitStore(self._bitarray)
+
+ def copy(self) -> BitStore:
+ return self if self.immutable else self._copy()
- def __getitem__(self, item: Union[int, slice], /) ->Union[int, BitStore]:
+ def __getitem__(self, item: Union[int, slice], /) -> Union[int, BitStore]:
+ # Use getindex or getslice instead
raise NotImplementedError
- def __len__(self) ->int:
- return (self.modified_length if self.modified_length is not None else
- len(self._bitarray))
+ def getindex_msb0(self, index: int, /) -> bool:
+ return bool(self._bitarray.__getitem__(index))
+
+ def getslice_withstep_msb0(self, key: slice, /) -> BitStore:
+ if self.modified_length is not None:
+ key = slice(*key.indices(self.modified_length))
+ return BitStore(self._bitarray.__getitem__(key))
+
+ def getslice_withstep_lsb0(self, key: slice, /) -> BitStore:
+ key = offset_slice_indices_lsb0(key, len(self))
+ return BitStore(self._bitarray.__getitem__(key))
+
+ def getslice_msb0(self, start: Optional[int], stop: Optional[int], /) -> BitStore:
+ if self.modified_length is not None:
+ key = slice(*slice(start, stop, None).indices(self.modified_length))
+ start = key.start
+ stop = key.stop
+ return BitStore(self._bitarray[start:stop])
+
+ def getslice_lsb0(self, start: Optional[int], stop: Optional[int], /) -> BitStore:
+ start, stop = offset_start_stop_lsb0(start, stop, len(self))
+ return BitStore(self._bitarray[start:stop])
+
+ def getindex_lsb0(self, index: int, /) -> bool:
+ return bool(self._bitarray.__getitem__(-index - 1))
+
+ @overload
+ def setitem_lsb0(self, key: int, value: int, /) -> None:
+ ...
+
+ @overload
+ def setitem_lsb0(self, key: slice, value: BitStore, /) -> None:
+ ...
+
+ def setitem_lsb0(self, key: Union[int, slice], value: Union[int, BitStore], /) -> None:
+ if isinstance(key, slice):
+ new_slice = offset_slice_indices_lsb0(key, len(self))
+ self._bitarray.__setitem__(new_slice, value._bitarray)
+ else:
+ self._bitarray.__setitem__(-key - 1, value)
+
+ def delitem_lsb0(self, key: Union[int, slice], /) -> None:
+ if isinstance(key, slice):
+ new_slice = offset_slice_indices_lsb0(key, len(self))
+ self._bitarray.__delitem__(new_slice)
+ else:
+ self._bitarray.__delitem__(-key - 1)
+
+ def invert_msb0(self, index: Optional[int] = None, /) -> None:
+ if index is not None:
+ self._bitarray.invert(index)
+ else:
+ self._bitarray.invert()
+
+ def invert_lsb0(self, index: Optional[int] = None, /) -> None:
+ if index is not None:
+ self._bitarray.invert(-index - 1)
+ else:
+ self._bitarray.invert()
+
+ def any_set(self) -> bool:
+ return self._bitarray.any()
+
+ def all_set(self) -> bool:
+ return self._bitarray.all()
+
+ def __len__(self) -> int:
+ return self.modified_length if self.modified_length is not None else len(self._bitarray)
+
+ def setitem_msb0(self, key, value, /):
+ if isinstance(value, BitStore):
+ self._bitarray.__setitem__(key, value._bitarray)
+ else:
+ self._bitarray.__setitem__(key, value)
+
+ def delitem_msb0(self, key, /):
+ self._bitarray.__delitem__(key)
diff --git a/bitstring/bitstore_helpers.py b/bitstring/bitstore_helpers.py
index 87a9787..41d30d2 100644
--- a/bitstring/bitstore_helpers.py
+++ b/bitstring/bitstore_helpers.py
@@ -1,4 +1,5 @@
from __future__ import annotations
+
import struct
import math
import functools
@@ -7,16 +8,263 @@ import bitarray
from bitstring.bitstore import BitStore
import bitstring
from bitstring.fp8 import p4binary_fmt, p3binary_fmt
-from bitstring.mxfp import e3m2mxfp_fmt, e2m3mxfp_fmt, e2m1mxfp_fmt, e4m3mxfp_saturate_fmt, e5m2mxfp_saturate_fmt, e4m3mxfp_overflow_fmt, e5m2mxfp_overflow_fmt
+from bitstring.mxfp import (e3m2mxfp_fmt, e2m3mxfp_fmt, e2m1mxfp_fmt, e4m3mxfp_saturate_fmt,
+ e5m2mxfp_saturate_fmt, e4m3mxfp_overflow_fmt, e5m2mxfp_overflow_fmt)
+
+# The size of various caches used to improve performance
CACHE_SIZE = 256
-def tidy_input_string(s: str) ->str:
+def tidy_input_string(s: str) -> str:
"""Return string made lowercase and with all whitespace and underscores removed."""
- pass
+ try:
+ t = s.split()
+ except (AttributeError, TypeError):
+ raise ValueError(f"Expected str object but received a {type(s)} with value {s}.")
+ return ''.join(t).lower().replace('_', '')
+
+
+@functools.lru_cache(CACHE_SIZE)
+def str_to_bitstore(s: str) -> BitStore:
+ _, tokens = bitstring.utils.tokenparser(s)
+ bs = BitStore()
+ for token in tokens:
+ bs += bitstore_from_token(*token)
+ bs.immutable = True
+ return bs
+
+
+def bin2bitstore(binstring: str) -> BitStore:
+ binstring = tidy_input_string(binstring)
+ binstring = binstring.replace('0b', '')
+ try:
+ return BitStore(binstring)
+ except ValueError:
+ raise bitstring.CreationError(f"Invalid character in bin initialiser {binstring}.")
+
+
+def bin2bitstore_unsafe(binstring: str) -> BitStore:
+ return BitStore(binstring)
+
+
+def hex2bitstore(hexstring: str) -> BitStore:
+ hexstring = tidy_input_string(hexstring)
+ hexstring = hexstring.replace('0x', '')
+ try:
+ ba = bitarray.util.hex2ba(hexstring)
+ except ValueError:
+ raise bitstring.CreationError("Invalid symbol in hex initialiser.")
+ return BitStore(ba)
+
+
+def oct2bitstore(octstring: str) -> BitStore:
+ octstring = tidy_input_string(octstring)
+ octstring = octstring.replace('0o', '')
+ try:
+ ba = bitarray.util.base2ba(8, octstring)
+ except ValueError:
+ raise bitstring.CreationError("Invalid symbol in oct initialiser.")
+ return BitStore(ba)
+
+
+def ue2bitstore(i: Union[str, int]) -> BitStore:
+ i = int(i)
+ if i < 0:
+ raise bitstring.CreationError("Cannot use negative initialiser for unsigned exponential-Golomb.")
+ if i == 0:
+ return BitStore('1')
+ tmp = i + 1
+ leadingzeros = -1
+ while tmp > 0:
+ tmp >>= 1
+ leadingzeros += 1
+ remainingpart = i + 1 - (1 << leadingzeros)
+ return BitStore('0' * leadingzeros + '1') + int2bitstore(remainingpart, leadingzeros, False)
+
+
+def se2bitstore(i: Union[str, int]) -> BitStore:
+ i = int(i)
+ if i > 0:
+ u = (i * 2) - 1
+ else:
+ u = -2 * i
+ return ue2bitstore(u)
+
+
+def uie2bitstore(i: Union[str, int]) -> BitStore:
+ i = int(i)
+ if i < 0:
+ raise bitstring.CreationError("Cannot use negative initialiser for unsigned interleaved exponential-Golomb.")
+ return BitStore('1' if i == 0 else '0' + '0'.join(bin(i + 1)[3:]) + '1')
+
+
+def sie2bitstore(i: Union[str, int]) -> BitStore:
+ i = int(i)
+ if i == 0:
+ return BitStore('1')
+ else:
+ return uie2bitstore(abs(i)) + (BitStore('1') if i < 0 else BitStore('0'))
+
+
+def bfloat2bitstore(f: Union[str, float], big_endian: bool) -> BitStore:
+ f = float(f)
+ fmt = '>f' if big_endian else '<f'
+ try:
+ b = struct.pack(fmt, f)
+ except OverflowError:
+ # For consistency we overflow to 'inf'.
+ b = struct.pack(fmt, float('inf') if f > 0 else float('-inf'))
+ return BitStore.frombytes(b[0:2]) if big_endian else BitStore.frombytes(b[2:4])
+
+
+def p4binary2bitstore(f: Union[str, float]) -> BitStore:
+ f = float(f)
+ u = p4binary_fmt.float_to_int8(f)
+ return int2bitstore(u, 8, False)
+
+
+def p3binary2bitstore(f: Union[str, float]) -> BitStore:
+ f = float(f)
+ u = p3binary_fmt.float_to_int8(f)
+ return int2bitstore(u, 8, False)
+
+
+def e4m3mxfp2bitstore(f: Union[str, float]) -> BitStore:
+ f = float(f)
+ if bitstring.options.mxfp_overflow == 'saturate':
+ u = e4m3mxfp_saturate_fmt.float_to_int(f)
+ else:
+ u = e4m3mxfp_overflow_fmt.float_to_int(f)
+ return int2bitstore(u, 8, False)
+
+
+def e5m2mxfp2bitstore(f: Union[str, float]) -> BitStore:
+ f = float(f)
+ if bitstring.options.mxfp_overflow == 'saturate':
+ u = e5m2mxfp_saturate_fmt.float_to_int(f)
+ else:
+ u = e5m2mxfp_overflow_fmt.float_to_int(f)
+ return int2bitstore(u, 8, False)
+
+
+def e3m2mxfp2bitstore(f: Union[str, float]) -> BitStore:
+ f = float(f)
+ if math.isnan(f):
+ raise ValueError("Cannot convert float('nan') to e3m2mxfp format as it has no representation for it.")
+ u = e3m2mxfp_fmt.float_to_int(f)
+ return int2bitstore(u, 6, False)
+
+
+def e2m3mxfp2bitstore(f: Union[str, float]) -> BitStore:
+ f = float(f)
+ if math.isnan(f):
+ raise ValueError("Cannot convert float('nan') to e2m3mxfp format as it has no representation for it.")
+ u = e2m3mxfp_fmt.float_to_int(f)
+ return int2bitstore(u, 6, False)
+
+
+def e2m1mxfp2bitstore(f: Union[str, float]) -> BitStore:
+ f = float(f)
+ if math.isnan(f):
+ raise ValueError("Cannot convert float('nan') to e2m1mxfp format as it has no representation for it.")
+ u = e2m1mxfp_fmt.float_to_int(f)
+ return int2bitstore(u, 4, False)
e8m0mxfp_allowed_values = [float(2 ** x) for x in range(-127, 128)]
-literal_bit_funcs: Dict[str, Callable[..., BitStore]] = {'0x': hex2bitstore,
- '0X': hex2bitstore, '0b': bin2bitstore, '0B': bin2bitstore, '0o':
- oct2bitstore, '0O': oct2bitstore}
+
+
+def e8m0mxfp2bitstore(f: Union[str, float]) -> BitStore:
+ f = float(f)
+ if math.isnan(f):
+ return BitStore('11111111')
+ try:
+ i = e8m0mxfp_allowed_values.index(f)
+ except ValueError:
+ raise ValueError(f"{f} is not a valid e8m0mxfp value. It must be exactly 2 ** i, for -127 <= i <= 127 or float('nan') as no rounding will be done.")
+ return int2bitstore(i, 8, False)
+
+
+def mxint2bitstore(f: Union[str, float]) -> BitStore:
+ f = float(f)
+ if math.isnan(f):
+ raise ValueError("Cannot convert float('nan') to mxint format as it has no representation for it.")
+ f *= 2 ** 6 # Remove the implicit scaling factor
+ if f > 127: # 1 + 63/64
+ return BitStore('01111111')
+ if f <= -128: # -2
+ return BitStore('10000000')
+ # Want to round to nearest, so move by 0.5 away from zero and round down by converting to int
+ if f >= 0.0:
+ f += 0.5
+ i = int(f)
+ # For ties-round-to-even
+ if f - i == 0.0 and i % 2:
+ i -= 1
+ else:
+ f -= 0.5
+ i = int(f)
+ if f - i == 0.0 and i % 2:
+ i += 1
+ return int2bitstore(i, 8, True)
+
+
+def int2bitstore(i: int, length: int, signed: bool) -> BitStore:
+ i = int(i)
+ try:
+ x = BitStore(bitarray.util.int2ba(i, length=length, endian='big', signed=signed))
+ except OverflowError as e:
+ if signed:
+ if i >= (1 << (length - 1)) or i < -(1 << (length - 1)):
+ raise bitstring.CreationError(f"{i} is too large a signed integer for a bitstring of length {length}. "
+ f"The allowed range is [{-(1 << (length - 1))}, {(1 << (length - 1)) - 1}].")
+ else:
+ if i >= (1 << length):
+ raise bitstring.CreationError(f"{i} is too large an unsigned integer for a bitstring of length {length}. "
+ f"The allowed range is [0, {(1 << length) - 1}].")
+ if i < 0:
+ raise bitstring.CreationError("uint cannot be initialised with a negative number.")
+ raise e
+ return x
+
+
+def intle2bitstore(i: int, length: int, signed: bool) -> BitStore:
+ x = int2bitstore(i, length, signed).tobytes()
+ return BitStore.frombytes(x[::-1])
+
+
+def float2bitstore(f: Union[str, float], length: int, big_endian: bool) -> BitStore:
+ f = float(f)
+ fmt = {16: '>e', 32: '>f', 64: '>d'}[length] if big_endian else {16: '<e', 32: '<f', 64: '<d'}[length]
+ try:
+ b = struct.pack(fmt, f)
+ except OverflowError:
+ # If float64 doesn't fit it automatically goes to 'inf'. This reproduces that behaviour for other types.
+ b = struct.pack(fmt, float('inf') if f > 0 else float('-inf'))
+ return BitStore.frombytes(b)
+
+
+literal_bit_funcs: Dict[str, Callable[..., BitStore]] = {
+ '0x': hex2bitstore,
+ '0X': hex2bitstore,
+ '0b': bin2bitstore,
+ '0B': bin2bitstore,
+ '0o': oct2bitstore,
+ '0O': oct2bitstore,
+}
+
+
+def bitstore_from_token(name: str, token_length: Optional[int], value: Optional[str]) -> BitStore:
+ if name in literal_bit_funcs:
+ return literal_bit_funcs[name](value)
+ try:
+ d = bitstring.dtypes.Dtype(name, token_length)
+ except ValueError as e:
+ raise bitstring.CreationError(f"Can't parse token: {e}")
+ if value is None and name != 'pad':
+ raise ValueError(f"Token {name} requires a value.")
+ bs = d.build(value)._bitstore
+ if token_length is not None and len(bs) != d.bitlength:
+ raise bitstring.CreationError(f"Token with length {token_length} packed with value of length {len(bs)} "
+ f"({name}:{token_length}={value}).")
+ return bs
diff --git a/bitstring/bitstream.py b/bitstring/bitstream.py
index 9a65360..8e7c6a1 100644
--- a/bitstring/bitstream.py
+++ b/bitstring/bitstream.py
@@ -1,11 +1,13 @@
from __future__ import annotations
+
import bitstring
from bitstring.bits import Bits, BitsType
from bitstring.dtypes import Dtype
from typing import Union, List, Any, Optional, overload, TypeVar, Tuple
import copy
import numbers
-TConstBitStream = TypeVar('TConstBitStream', bound='ConstBitStream')
+
+TConstBitStream = TypeVar("TConstBitStream", bound='ConstBitStream')
class ConstBitStream(Bits):
@@ -54,11 +56,11 @@ class ConstBitStream(Bits):
len -- Length of the bitstring in bits.
pos -- The current bit position in the bitstring.
"""
- __slots__ = '_pos',
- def __init__(self, auto: Optional[Union[BitsType, int]]=None, /, length:
- Optional[int]=None, offset: Optional[int]=None, pos: int=0, **kwargs
- ) ->None:
+ __slots__ = ('_pos',)
+
+ def __init__(self, auto: Optional[Union[BitsType, int]] = None, /, length: Optional[int] = None,
+ offset: Optional[int] = None, pos: int = 0, **kwargs) -> None:
"""Either specify an 'auto' initialiser:
A string of comma separated tokens, an integer, a file object,
a bytearray, a boolean iterable or another bitstring.
@@ -101,36 +103,47 @@ class ConstBitStream(Bits):
if pos < 0:
pos += len(self._bitstore)
if pos < 0 or pos > len(self._bitstore):
- raise bitstring.CreationError(
- f'Cannot set pos to {pos} when length is {len(self._bitstore)}.'
- )
+ raise bitstring.CreationError(f"Cannot set pos to {pos} when length is {len(self._bitstore)}.")
self._pos = pos
self._bitstore.immutable = True
- def _setbytepos(self, bytepos: int) ->None:
+ def _setbytepos(self, bytepos: int) -> None:
"""Move to absolute byte-aligned position in stream."""
- pass
+ self._setbitpos(bytepos * 8)
- def _getbytepos(self) ->int:
+ def _getbytepos(self) -> int:
"""Return the current position in the stream in bytes. Must be byte aligned."""
- pass
+ if self._pos % 8:
+ raise bitstring.ByteAlignError("Not byte aligned when using bytepos property.")
+ return self._pos // 8
- def _setbitpos(self, pos: int) ->None:
+ def _setbitpos(self, pos: int) -> None:
"""Move to absolute position bit in bitstream."""
- pass
+ if pos < 0:
+ raise ValueError("Bit position cannot be negative.")
+ if pos > len(self):
+ raise ValueError("Cannot seek past the end of the data.")
+ self._pos = pos
- def _getbitpos(self) ->int:
+ def _getbitpos(self) -> int:
"""Return the current position in the stream in bits."""
- pass
+ return self._pos
- def __copy__(self: TConstBitStream) ->TConstBitStream:
+ def _clear(self) -> None:
+ Bits._clear(self)
+ self._pos = 0
+
+ def __copy__(self: TConstBitStream) -> TConstBitStream:
"""Return a new copy of the ConstBitStream for the copy module."""
+ # Note that if you want a new copy (different ID), use _copy instead.
+ # The copy can use the same datastore as it's immutable.
s = self.__class__()
s._bitstore = self._bitstore
+ # Reset the bit position, don't copy it.
s._pos = 0
return s
- def __and__(self: TConstBitStream, bs: BitsType, /) ->TConstBitStream:
+ def __and__(self: TConstBitStream, bs: BitsType, /) -> TConstBitStream:
"""Bit-wise 'and' between two bitstrings. Returns new bitstring.
bs -- The bitstring to '&' with.
@@ -142,7 +155,7 @@ class ConstBitStream(Bits):
s._pos = 0
return s
- def __or__(self: TConstBitStream, bs: BitsType, /) ->TConstBitStream:
+ def __or__(self: TConstBitStream, bs: BitsType, /) -> TConstBitStream:
"""Bit-wise 'or' between two bitstrings. Returns new bitstring.
bs -- The bitstring to '|' with.
@@ -154,7 +167,7 @@ class ConstBitStream(Bits):
s._pos = 0
return s
- def __xor__(self: TConstBitStream, bs: BitsType, /) ->TConstBitStream:
+ def __xor__(self: TConstBitStream, bs: BitsType, /) -> TConstBitStream:
"""Bit-wise 'xor' between two bitstrings. Returns new bitstring.
bs -- The bitstring to '^' with.
@@ -166,7 +179,7 @@ class ConstBitStream(Bits):
s._pos = 0
return s
- def __add__(self: TConstBitStream, bs: BitsType, /) ->TConstBitStream:
+ def __add__(self: TConstBitStream, bs: BitsType, /) -> TConstBitStream:
"""Concatenate bitstrings and return new bitstring.
bs -- the bitstring to append.
@@ -176,7 +189,7 @@ class ConstBitStream(Bits):
s._pos = 0
return s
- def append(self, bs: BitsType, /) ->None:
+ def append(self, bs: BitsType, /) -> None:
"""Append a bitstring to the current bitstring.
bs -- The bitstring to append.
@@ -184,9 +197,10 @@ class ConstBitStream(Bits):
The current bit position will be moved to the end of the BitStream.
"""
- pass
+ self._append(bs)
+ self._pos = len(self)
- def __repr__(self) ->str:
+ def __repr__(self) -> str:
"""Return representation that could be used to recreate the bitstring.
If the returned string is too long it will be truncated. See __str__().
@@ -194,7 +208,7 @@ class ConstBitStream(Bits):
"""
return self._repr(self.__class__.__name__, len(self), self._pos)
- def overwrite(self, bs: BitsType, /, pos: Optional[int]=None) ->None:
+ def overwrite(self, bs: BitsType, /, pos: Optional[int] = None) -> None:
"""Overwrite with bitstring at bit position pos.
bs -- The bitstring to overwrite with.
@@ -204,11 +218,20 @@ class ConstBitStream(Bits):
Raises ValueError if pos < 0 or pos > len(self).
"""
- pass
-
- def find(self, bs: BitsType, /, start: Optional[int]=None, end:
- Optional[int]=None, bytealigned: Optional[bool]=None) ->Union[Tuple
- [int], Tuple[()]]:
+ bs = Bits._create_from_bitstype(bs)
+ if len(bs) == 0:
+ return
+ if pos is None:
+ pos = self._pos
+ if pos < 0:
+ pos += len(self)
+ if pos < 0 or pos > len(self):
+ raise ValueError("Overwrite starts outside boundary of bitstring.")
+ self._overwrite(bs, pos)
+ self._pos = pos + len(bs)
+
+ def find(self, bs: BitsType, /, start: Optional[int] = None, end: Optional[int] = None,
+ bytealigned: Optional[bool] = None) -> Union[Tuple[int], Tuple[()]]:
"""Find first occurrence of substring bs.
Returns a single item tuple with the bit position if found, or an
@@ -229,11 +252,14 @@ class ConstBitStream(Bits):
(6,)
"""
- pass
- def rfind(self, bs: BitsType, /, start: Optional[int]=None, end:
- Optional[int]=None, bytealigned: Optional[bool]=None) ->Union[Tuple
- [int], Tuple[()]]:
+ p = super().find(bs, start, end, bytealigned)
+ if p:
+ self._pos = p[0]
+ return p
+
+ def rfind(self, bs: BitsType, /, start: Optional[int] = None, end: Optional[int] = None,
+ bytealigned: Optional[bool] = None) -> Union[Tuple[int], Tuple[()]]:
"""Find final occurrence of substring bs.
Returns a single item tuple with the bit position if found, or an
@@ -251,10 +277,20 @@ class ConstBitStream(Bits):
if end < start.
"""
- pass
+ p = super().rfind(bs, start, end, bytealigned)
+ if p:
+ self._pos = p[0]
+ return p
+
+ @overload
+ def read(self, fmt: int) -> Bits:
+ ...
+
+ @overload
+ def read(self, fmt: str) -> Any:
+ ...
- def read(self, fmt: Union[int, str, Dtype]) ->Union[int, float, str,
- Bits, bool, bytes, None]:
+ def read(self, fmt: Union[int, str, Dtype]) -> Union[int, float, str, Bits, bool, bytes, None]:
"""Interpret next bits according to the format string and return result.
fmt -- Token string describing how to interpret the next bits.
@@ -290,10 +326,38 @@ class ConstBitStream(Bits):
Raises ValueError if the format is not understood.
"""
- pass
-
- def readlist(self, fmt: Union[str, List[Union[int, str, Dtype]]], **kwargs
- ) ->List[Union[int, float, str, Bits, bool, bytes, None]]:
+ p = self._pos
+ if isinstance(fmt, numbers.Integral):
+ if fmt < 0:
+ raise ValueError("Cannot read negative amount.")
+ if fmt > len(self) - self._pos:
+ raise bitstring.ReadError(f"Cannot read {fmt} bits, only {len(self) - self._pos} available.")
+ bs = self._slice(self._pos, self._pos + fmt)
+ self._pos += fmt
+ return bs
+ dtype = bitstring.dtypes.Dtype(fmt)
+ if dtype.bitlength is None and not dtype.variable_length:
+ # No length specified? Try again, but read to end.
+ bitlength = len(self) - self._pos
+ items, remainder = divmod(bitlength, dtype.bits_per_item)
+ if remainder != 0:
+ raise ValueError(
+ f"The '{dtype.name}' type must have a bit length that is a multiple of {dtype.bits_per_item}"
+ f" so cannot be read from the {bitlength} bits that are available.")
+ dtype = bitstring.dtypes.Dtype(fmt, items)
+ if dtype.bitlength is not None:
+ val = dtype.read_fn(self, self._pos)
+ self._pos += dtype.bitlength
+ else:
+ val, self._pos = dtype.read_fn(self, self._pos)
+
+ if self._pos > len(self):
+ self._pos = p
+ raise bitstring.ReadError(f"Reading off end of bitstring with fmt '{fmt}'. Only {len(self) - p} bits available.")
+ return val
+
+ def readlist(self, fmt: Union[str, List[Union[int, str, Dtype]]], **kwargs) \
+ -> List[Union[int, float, str, Bits, bool, bytes, None]]:
"""Interpret next bits according to format string(s) and return list.
fmt -- A single string or list of strings with comma separated tokens
@@ -314,10 +378,10 @@ class ConstBitStream(Bits):
>>> i, bs1, bs2 = s.readlist(['uint:12', 10, 10])
"""
- pass
+ value, self._pos = self._readlist(fmt, self._pos, **kwargs)
+ return value
- def readto(self: TConstBitStream, bs: BitsType, /, bytealigned:
- Optional[bool]=None) ->TConstBitStream:
+ def readto(self: TConstBitStream, bs: BitsType, /, bytealigned: Optional[bool] = None) -> TConstBitStream:
"""Read up to and including next occurrence of bs and return result.
bs -- The bitstring to find.
@@ -328,10 +392,25 @@ class ConstBitStream(Bits):
Raises ReadError if bs is not found.
"""
- pass
+ if isinstance(bs, numbers.Integral):
+ raise ValueError("Integers cannot be searched for")
+ bs = Bits._create_from_bitstype(bs)
+ oldpos = self._pos
+ p = self.find(bs, self._pos, bytealigned=bytealigned)
+ if not p:
+ raise bitstring.ReadError("Substring not found")
+ self._pos += len(bs)
+ return self._slice(oldpos, self._pos)
+
+ @overload
+ def peek(self: TConstBitStream, fmt: int) -> TConstBitStream:
+ ...
- def peek(self: TConstBitStream, fmt: Union[int, str]) ->Union[int,
- float, str, TConstBitStream, bool, bytes, None]:
+ @overload
+ def peek(self, fmt: str) -> Union[int, float, str, TConstBitStream, bool, bytes, None]:
+ ...
+
+ def peek(self: TConstBitStream, fmt: Union[int, str]) -> Union[int, float, str, TConstBitStream, bool, bytes, None]:
"""Interpret next bits according to format string and return result.
fmt -- Token string describing how to interpret the next bits.
@@ -345,10 +424,13 @@ class ConstBitStream(Bits):
See the docstring for 'read' for token examples.
"""
- pass
+ pos_before = self._pos
+ value = self.read(fmt)
+ self._pos = pos_before
+ return value
- def peeklist(self, fmt: Union[str, List[Union[int, str]]], **kwargs
- ) ->List[Union[int, float, str, Bits, None]]:
+ def peeklist(self, fmt: Union[str, List[Union[int, str]]], **kwargs) \
+ -> List[Union[int, float, str, Bits, None]]:
"""Interpret next bits according to format string(s) and return list.
fmt -- One or more integers or strings with comma separated tokens describing
@@ -365,27 +447,38 @@ class ConstBitStream(Bits):
See the docstring for 'read' for token examples.
"""
- pass
+ pos = self._pos
+ return_values = self.readlist(fmt, **kwargs)
+ self._pos = pos
+ return return_values
- def bytealign(self) ->int:
+ def bytealign(self) -> int:
"""Align to next byte and return number of skipped bits.
Raises ValueError if the end of the bitstring is reached before
aligning to the next byte.
"""
- pass
+ skipped = (8 - (self._pos % 8)) % 8
+ self.pos += skipped
+ return skipped
+
+ @classmethod
+ def fromstring(cls: TBits, s: str, /) -> TBits:
+ x = super().fromstring(s)
+ x._pos = 0
+ x._bitstore.immutable = True
+ return x
@overload
- def __getitem__(self: TBits, key: slice, /) ->TBits:
+ def __getitem__(self: TBits, key: slice, /) -> TBits:
...
@overload
- def __getitem__(self: TBits, key: int, /) ->bool:
+ def __getitem__(self: TBits, key: int, /) -> bool:
...
- def __getitem__(self: TBits, key: Union[slice, int], /) ->Union[TBits, bool
- ]:
+ def __getitem__(self: TBits, key: Union[slice, int], /) -> Union[TBits, bool]:
"""Return a new bitstring representing a slice of the current bitstring."""
if isinstance(key, numbers.Integral):
return bool(self._bitstore.getindex(key))
@@ -393,18 +486,16 @@ class ConstBitStream(Bits):
bs._bitstore = self._bitstore.getslice_withstep(key)
bs._pos = 0
return bs
- pos = property(_getbitpos, _setbitpos, doc=
- """The position in the bitstring in bits. Read and write.
- """
- )
- bitpos = property(_getbitpos, _setbitpos, doc=
- """The position in the bitstring in bits. Read and write.
- """
- )
- bytepos = property(_getbytepos, _setbytepos, doc=
- """The position in the bitstring in bytes. Read and write.
- """
- )
+
+ pos = property(_getbitpos, _setbitpos,
+ doc="""The position in the bitstring in bits. Read and write.
+ """)
+ bitpos = property(_getbitpos, _setbitpos,
+ doc="""The position in the bitstring in bits. Read and write.
+ """)
+ bytepos = property(_getbytepos, _setbytepos,
+ doc="""The position in the bitstring in bytes. Read and write.
+ """)
class BitStream(ConstBitStream, bitstring.BitArray):
@@ -464,11 +555,11 @@ class BitStream(ConstBitStream, bitstring.BitArray):
len -- Length of the bitstring in bits.
pos -- The current bit position in the bitstring.
"""
+
__slots__ = ()
- def __init__(self, auto: Optional[Union[BitsType, int]]=None, /, length:
- Optional[int]=None, offset: Optional[int]=None, pos: int=0, **kwargs
- ) ->None:
+ def __init__(self, auto: Optional[Union[BitsType, int]] = None, /, length: Optional[int] = None,
+ offset: Optional[int] = None, pos: int = 0, **kwargs) -> None:
"""Either specify an 'auto' initialiser:
A string of comma separated tokens, an integer, a file object,
a bytearray, a boolean iterable or another bitstring.
@@ -513,14 +604,14 @@ class BitStream(ConstBitStream, bitstring.BitArray):
self._bitstore = self._bitstore._copy()
self._bitstore.immutable = False
- def __copy__(self) ->BitStream:
+ def __copy__(self) -> BitStream:
"""Return a new copy of the BitStream."""
s_copy = object.__new__(BitStream)
s_copy._pos = 0
s_copy._bitstore = self._bitstore.copy()
return s_copy
- def __iadd__(self, bs: BitsType, /) ->BitStream:
+ def __iadd__(self, bs: BitsType, /) -> BitStream:
"""Append to current bitstring. Return self.
bs -- the bitstring to append.
@@ -531,22 +622,24 @@ class BitStream(ConstBitStream, bitstring.BitArray):
self._pos = len(self)
return self
- def prepend(self, bs: BitsType, /) ->None:
+ def prepend(self, bs: BitsType, /) -> None:
"""Prepend a bitstring to the current bitstring.
bs -- The bitstring to prepend.
"""
- pass
+ bs = Bits._create_from_bitstype(bs)
+ super().prepend(bs)
+ self._pos = 0
- def __setitem__(self, /, key: Union[slice, int], value: BitsType) ->None:
+ def __setitem__(self, /, key: Union[slice, int], value: BitsType) -> None:
length_before = len(self)
super().__setitem__(key, value)
if len(self) != length_before:
self._pos = 0
return
- def __delitem__(self, /, key: Union[slice, int]) ->None:
+ def __delitem__(self, /, key: Union[slice, int]) -> None:
"""Delete item or range.
>>> a = BitStream('0x001122')
@@ -560,7 +653,7 @@ class BitStream(ConstBitStream, bitstring.BitArray):
if len(self) != length_before:
self._pos = 0
- def insert(self, bs: BitsType, /, pos: Optional[int]=None) ->None:
+ def insert(self, bs: BitsType, /, pos: Optional[int] = None) -> None:
"""Insert bitstring at bit position pos.
bs -- The bitstring to insert.
@@ -570,11 +663,22 @@ class BitStream(ConstBitStream, bitstring.BitArray):
Raises ValueError if pos < 0 or pos > len(self).
"""
- pass
-
- def replace(self, old: BitsType, new: BitsType, start: Optional[int]=
- None, end: Optional[int]=None, count: Optional[int]=None,
- bytealigned: Optional[bool]=None) ->int:
+ bs = Bits._create_from_bitstype(bs)
+ if len(bs) == 0:
+ return
+ if bs is self:
+ bs = self._copy()
+ if pos is None:
+ pos = self._pos
+ if pos < 0:
+ pos += len(self)
+ if not 0 <= pos <= len(self):
+ raise ValueError("Invalid insert position.")
+ self._insert(bs, pos)
+ self._pos = pos + len(bs)
+
+ def replace(self, old: BitsType, new: BitsType, start: Optional[int] = None, end: Optional[int] = None,
+ count: Optional[int] = None, bytealigned: Optional[bool] = None) -> int:
"""Replace all occurrences of old with new in place.
Returns number of replacements made.
@@ -594,4 +698,17 @@ class BitStream(ConstBitStream, bitstring.BitArray):
out of range.
"""
- pass
+ if count == 0:
+ return 0
+ if len(old := Bits._create_from_bitstype(old)) == 0:
+ raise ValueError("Empty bitstring cannot be replaced.")
+ start, end = self._validate_slice(start, end)
+ new = Bits._create_from_bitstype(new)
+ if new is self:
+ # Prevent self assignment woes
+ new = copy.copy(self)
+ length_before = len(self)
+ replacement_count = self._replace(old, new, start, end, 0 if count is None else count, bytealigned)
+ if len(self) != length_before:
+ self._pos = 0
+ return replacement_count
\ No newline at end of file
diff --git a/bitstring/bitstring_options.py b/bitstring/bitstring_options.py
index 8928d8e..de84ae5 100644
--- a/bitstring/bitstring_options.py
+++ b/bitstring/bitstring_options.py
@@ -1,25 +1,80 @@
from __future__ import annotations
+
import bitstring
import os
class Options:
"""Internal class to create singleton module options instance."""
+
_instance = None
def __init__(self):
self.set_lsb0(False)
self._bytealigned = False
self.mxfp_overflow = 'saturate'
+
self.no_color = False
no_color = os.getenv('NO_COLOR')
self.no_color = True if no_color else False
- def __repr__(self) ->str:
- attributes = {attr: getattr(self, attr) for attr in dir(self) if
- not attr.startswith('_') and not callable(getattr(self, attr))}
- return '\n'.join(f'{attr}: {value!r}' for attr, value in attributes
- .items())
+ @property
+ def mxfp_overflow(self) -> str:
+ return self._mxfp_overflow
+
+ @mxfp_overflow.setter
+ def mxfp_overflow(self, value: str) -> None:
+ allowed_values = ('saturate', 'overflow')
+ if value not in allowed_values:
+ raise ValueError(f"mxfp_overflow must be one of {allowed_values}, not {value}.")
+ self._mxfp_overflow = value
+
+ def __repr__(self) -> str:
+ attributes = {attr: getattr(self, attr) for attr in dir(self) if not attr.startswith('_') and not callable(getattr(self, attr))}
+ return '\n'.join(f"{attr}: {value!r}" for attr, value in attributes.items())
+
+ @property
+ def lsb0(self) -> bool:
+ return self._lsb0
+
+ @lsb0.setter
+ def lsb0(self, value: bool) -> None:
+ self.set_lsb0(value)
+
+ def set_lsb0(self, value: bool) -> None:
+ self._lsb0 = bool(value)
+ Bits = bitstring.bits.Bits
+ BitArray = bitstring.bitarray_.BitArray
+ BitStore = bitstring.bitstore.BitStore
+
+ lsb0_methods = {
+ Bits: {'_find': Bits._find_lsb0, '_rfind': Bits._rfind_lsb0, '_findall': Bits._findall_lsb0},
+ BitArray: {'_ror': BitArray._rol_msb0, '_rol': BitArray._ror_msb0, '_append': BitArray._append_lsb0,
+ '_prepend': BitArray._append_msb0},
+ BitStore: {'__setitem__': BitStore.setitem_lsb0, '__delitem__': BitStore.delitem_lsb0,
+ 'getindex': BitStore.getindex_lsb0, 'getslice': BitStore.getslice_lsb0,
+ 'getslice_withstep': BitStore.getslice_withstep_lsb0, 'invert': BitStore.invert_lsb0}
+ }
+ msb0_methods = {
+ Bits: {'_find': Bits._find_msb0, '_rfind': Bits._rfind_msb0, '_findall': Bits._findall_msb0},
+ BitArray: {'_ror': BitArray._ror_msb0, '_rol': BitArray._rol_msb0, '_append': BitArray._append_msb0,
+ '_prepend': BitArray._append_lsb0},
+ BitStore: {'__setitem__': BitStore.setitem_msb0, '__delitem__': BitStore.delitem_msb0,
+ 'getindex': BitStore.getindex_msb0, 'getslice': BitStore.getslice_msb0,
+ 'getslice_withstep': BitStore.getslice_withstep_msb0, 'invert': BitStore.invert_msb0}
+ }
+ methods = lsb0_methods if self._lsb0 else msb0_methods
+ for cls, method_dict in methods.items():
+ for attr, method in method_dict.items():
+ setattr(cls, attr, method)
+
+ @property
+ def bytealigned(self) -> bool:
+ return self._bytealigned
+
+ @bytealigned.setter
+ def bytealigned(self, value: bool) -> None:
+ self._bytealigned = bool(value)
def __new__(cls):
if cls._instance is None:
@@ -28,14 +83,13 @@ class Options:
class Colour:
-
- def __new__(cls, use_colour: bool) ->Colour:
+ def __new__(cls, use_colour: bool) -> Colour:
x = super().__new__(cls)
if use_colour:
- cls.blue = '\x1b[34m'
- cls.purple = '\x1b[35m'
- cls.green = '\x1b[32m'
- cls.off = '\x1b[0m'
+ cls.blue = '\033[34m'
+ cls.purple = '\033[35m'
+ cls.green = '\033[32m'
+ cls.off = '\033[0m'
else:
cls.blue = cls.purple = cls.green = cls.off = ''
return x
diff --git a/bitstring/dtypes.py b/bitstring/dtypes.py
index 775063f..86c4ad1 100644
--- a/bitstring/dtypes.py
+++ b/bitstring/dtypes.py
@@ -1,12 +1,36 @@
from __future__ import annotations
+
import functools
from typing import Optional, Dict, Any, Union, Tuple, Callable
import inspect
import bitstring
from bitstring import utils
+
CACHE_SIZE = 256
+def scaled_get_fn(get_fn, s: Union[int, float]):
+ def wrapper(*args, scale=s, **kwargs):
+ return get_fn(*args, **kwargs) * scale
+ return wrapper
+
+
+def scaled_set_fn(set_fn, s: Union[int, float]):
+ def wrapper(bs, value, *args, scale=s, **kwargs):
+ return set_fn(bs, value / scale, *args, **kwargs)
+ return wrapper
+
+
+def scaled_read_fn(read_fn, s: Union[int, float]):
+ def wrapper(*args, scale=s, **kwargs):
+ val = read_fn(*args, **kwargs)
+ if isinstance(val, tuple):
+ val, pos = val
+ return val * scale, pos
+ return val * scale
+ return wrapper
+
+
class Dtype:
"""A data type class, representing a concrete interpretation of binary data.
@@ -17,6 +41,7 @@ class Dtype:
>>> mxfp = Dtype('e3m2mxfp', scale=2 ** 6) # dtype with scaling factor
"""
+
_name: str
_read_fn: Callable
_set_fn: Callable
@@ -30,8 +55,7 @@ class Dtype:
_length: Optional[int]
_scale: Union[None, float, int]
- def __new__(cls, token: Union[str, Dtype], /, length: Optional[int]=
- None, scale: Union[None, float, int]=None) ->Dtype:
+ def __new__(cls, token: Union[str, Dtype], /, length: Optional[int] = None, scale: Union[None, float, int] = None) -> Dtype:
if isinstance(token, cls):
return token
if length is None:
@@ -42,92 +66,144 @@ class Dtype:
return x
@property
- def scale(self) ->Union[int, float, None]:
+ def scale(self) -> Union[int, float, None]:
"""The multiplicative scale applied when interpreting the data."""
- pass
+ return self._scale
@property
- def name(self) ->str:
+ def name(self) -> str:
"""A string giving the name of the data type."""
- pass
+ return self._name
@property
- def length(self) ->int:
+ def length(self) -> int:
"""The length of the data type in units of bits_per_item. Set to None for variable length dtypes."""
- pass
+ return self._length
@property
- def bitlength(self) ->Optional[int]:
+ def bitlength(self) -> Optional[int]:
"""The number of bits needed to represent a single instance of the data type. Set to None for variable length dtypes."""
- pass
+ return self._bitlength
@property
- def bits_per_item(self) ->int:
+ def bits_per_item(self) -> int:
"""The number of bits for each unit of length. Usually 1, but equals 8 for bytes type."""
- pass
+ return self._bits_per_item
@property
- def variable_length(self) ->bool:
+ def variable_length(self) -> bool:
"""If True then the length of the data type depends on the data being interpreted, and must not be specified."""
- pass
+ return self._variable_length
@property
- def return_type(self) ->Any:
+ def return_type(self) -> Any:
"""The type of the value returned by the parse method, such as int, float or str."""
- pass
+ return self._return_type
@property
- def is_signed(self) ->bool:
+ def is_signed(self) -> bool:
"""If True then the data type represents a signed quantity."""
- pass
+ return self._is_signed
@property
- def set_fn(self) ->Optional[Callable]:
+ def set_fn(self) -> Optional[Callable]:
"""A function to set the value of the data type."""
- pass
+ return self._set_fn
@property
- def get_fn(self) ->Callable:
+ def get_fn(self) -> Callable:
"""A function to get the value of the data type."""
- pass
+ return self._get_fn
@property
- def read_fn(self) ->Callable:
+ def read_fn(self) -> Callable:
"""A function to read the value of the data type."""
- pass
+ return self._read_fn
+
+ def _set_scale(self, value: Union[None, float, int]) -> None:
+ self._scale = value
+ if self._scale is None:
+ return
+ if self._scale == 0:
+ raise ValueError("A Dtype's scale factor must not be zero.")
+ if not hasattr(self, 'unscaled_get_fn'):
+ self.unscaled_get_fn = self._get_fn
+ self.unscaled_set_fn = self._set_fn
+ self.unscaled_read_fn = self._read_fn
+ self._get_fn = scaled_get_fn(self.unscaled_get_fn, self._scale)
+ self._set_fn = scaled_set_fn(self.unscaled_set_fn, self._scale)
+ self._read_fn = scaled_read_fn(self.unscaled_read_fn, self._scale)
- def __hash__(self) ->int:
+ @classmethod
+ @functools.lru_cache(CACHE_SIZE)
+ def _new_from_token(cls, token: str, scale: Union[None, float, int] = None) -> Dtype:
+ token = ''.join(token.split())
+ return dtype_register.get_dtype(*utils.parse_name_length_token(token), scale=scale)
+
+ def __hash__(self) -> int:
return hash((self._name, self._length))
- def build(self, value: Any, /) ->bitstring.Bits:
+ @classmethod
+ @functools.lru_cache(CACHE_SIZE)
+ def _create(cls, definition: DtypeDefinition, length: Optional[int], scale: Union[None, float, int]) -> Dtype:
+ x = super().__new__(cls)
+ x._name = definition.name
+ x._bitlength = x._length = length
+ x._bits_per_item = definition.multiplier
+ if x._bitlength is not None:
+ x._bitlength *= x._bits_per_item
+ x._set_fn_needs_length = definition.set_fn_needs_length
+ x._variable_length = definition.variable_length
+ if x._variable_length or dtype_register.names[x._name].allowed_lengths.only_one_value():
+ x._read_fn = definition.read_fn
+ else:
+ x._read_fn = functools.partial(definition.read_fn, length=x._bitlength)
+ if definition.set_fn is None:
+ x._set_fn = None
+ else:
+ if x._set_fn_needs_length:
+ x._set_fn = functools.partial(definition.set_fn, length=x._bitlength)
+ else:
+ x._set_fn = definition.set_fn
+ x._get_fn = definition.get_fn
+ x._return_type = definition.return_type
+ x._is_signed = definition.is_signed
+ x._set_scale(scale)
+ return x
+
+ def build(self, value: Any, /) -> bitstring.Bits:
"""Create a bitstring from a value.
The value parameter should be of a type appropriate to the dtype.
"""
- pass
+ b = bitstring.Bits()
+ self._set_fn(b, value)
+ if self.bitlength is not None and len(b) != self.bitlength:
+ raise ValueError(f"Dtype has a length of {self.bitlength} bits, but value '{value}' has {len(b)} bits.")
+ return b
- def parse(self, b: BitsType, /) ->Any:
+ def parse(self, b: BitsType, /) -> Any:
"""Parse a bitstring to find its value.
The b parameter should be a bitstring of the appropriate length, or an object that can be converted to a bitstring."""
- pass
+ b = bitstring.Bits._create_from_bitstype(b)
+ return self._get_fn(bitstring.Bits(b))
- def __str__(self) ->str:
+ def __str__(self) -> str:
if self._scale is not None:
return self.__repr__()
- hide_length = self._variable_length or dtype_register.names[self._name
- ].allowed_lengths.only_one_value() or self._length is None
+ hide_length = self._variable_length or dtype_register.names[self._name].allowed_lengths.only_one_value() or self._length is None
length_str = '' if hide_length else str(self._length)
- return f'{self._name}{length_str}'
+ return f"{self._name}{length_str}"
- def __repr__(self) ->str:
- hide_length = self._variable_length or dtype_register.names[self._name
- ].allowed_lengths.only_one_value() or self._length is None
+ def __repr__(self) -> str:
+ hide_length = self._variable_length or dtype_register.names[self._name].allowed_lengths.only_one_value() or self._length is None
length_str = '' if hide_length else ', ' + str(self._length)
if self._scale is None:
scale_str = ''
else:
try:
+ # This will only succeed for powers of two from -127 to 127.
e8m0 = bitstring.Bits(e8m0mxfp=self._scale)
except ValueError:
scale_str = f', scale={self._scale}'
@@ -137,105 +213,94 @@ class Dtype:
scale_str = f', scale={self._scale}'
else:
scale_str = f', scale=2 ** {power_of_two}'
- return (
- f"{self.__class__.__name__}('{self._name}'{length_str}{scale_str})"
- )
+ return f"{self.__class__.__name__}('{self._name}'{length_str}{scale_str})"
- def __eq__(self, other: Any) ->bool:
+ def __eq__(self, other: Any) -> bool:
if isinstance(other, Dtype):
return self._name == other._name and self._length == other._length
return False
class AllowedLengths:
-
- def __init__(self, value: Tuple[int, ...]=tuple()) ->None:
+ def __init__(self, value: Tuple[int, ...] = tuple()) -> None:
if len(value) >= 3 and value[-1] is Ellipsis:
step = value[1] - value[0]
for i in range(1, len(value) - 1):
if value[i] - value[i - 1] != step:
- raise ValueError(
- f'Allowed length tuples must be equally spaced when final element is Ellipsis, but got {value}.'
- )
- self.values = value[0], value[1], Ellipsis
+ raise ValueError(f"Allowed length tuples must be equally spaced when final element is Ellipsis, but got {value}.")
+ self.values = (value[0], value[1], Ellipsis)
else:
self.values = value
- def __str__(self) ->str:
+ def __str__(self) -> str:
if self.values and self.values[-1] is Ellipsis:
- return f'({self.values[0]}, {self.values[1]}, ...)'
+ return f"({self.values[0]}, {self.values[1]}, ...)"
return str(self.values)
- def __contains__(self, other: Any) ->bool:
+ def __contains__(self, other: Any) -> bool:
if not self.values:
return True
if self.values[-1] is Ellipsis:
- return (other - self.values[0]) % (self.values[1] - self.values[0]
- ) == 0
+ return (other - self.values[0]) % (self.values[1] - self.values[0]) == 0
return other in self.values
+ def only_one_value(self) -> bool:
+ return self.values and len(self.values) == 1
+
class DtypeDefinition:
"""Represents a class of dtypes, such as uint or float, rather than a concrete dtype such as uint8.
Not (yet) part of the public interface."""
- def __init__(self, name: str, set_fn, get_fn, return_type: Any=Any,
- is_signed: bool=False, bitlength2chars_fn=None, variable_length:
- bool=False, allowed_lengths: Tuple[int, ...]=tuple(), multiplier:
- int=1, description: str=''):
+ def __init__(self, name: str, set_fn, get_fn, return_type: Any = Any, is_signed: bool = False, bitlength2chars_fn=None,
+ variable_length: bool = False, allowed_lengths: Tuple[int, ...] = tuple(), multiplier: int = 1, description: str = ''):
+
+ # Consistency checks
if int(multiplier) != multiplier or multiplier <= 0:
- raise ValueError('multiplier must be an positive integer')
+ raise ValueError("multiplier must be an positive integer")
if variable_length and allowed_lengths:
- raise ValueError(
- "A variable length dtype can't have allowed lengths.")
- if (variable_length and set_fn is not None and 'length' in inspect.
- signature(set_fn).parameters):
- raise ValueError(
- "A variable length dtype can't have a set_fn which takes a length."
- )
+ raise ValueError("A variable length dtype can't have allowed lengths.")
+ if variable_length and set_fn is not None and 'length' in inspect.signature(set_fn).parameters:
+ raise ValueError("A variable length dtype can't have a set_fn which takes a length.")
+
self.name = name
self.description = description
self.return_type = return_type
self.is_signed = is_signed
self.variable_length = variable_length
self.allowed_lengths = AllowedLengths(allowed_lengths)
+
self.multiplier = multiplier
- self.set_fn_needs_length = (set_fn is not None and 'length' in
- inspect.signature(set_fn).parameters)
+
+ # Can work out if set_fn needs length based on its signature.
+ self.set_fn_needs_length = set_fn is not None and 'length' in inspect.signature(set_fn).parameters
self.set_fn = set_fn
- if self.allowed_lengths.values:
+ if self.allowed_lengths.values:
def allowed_length_checked_get_fn(bs):
if len(bs) not in self.allowed_lengths:
if self.allowed_lengths.only_one_value():
- raise bitstring.InterpretError(
- f"'{self.name}' dtypes must have a length of {self.allowed_lengths.values[0]}, but received a length of {len(bs)}."
- )
+ raise bitstring.InterpretError(f"'{self.name}' dtypes must have a length of {self.allowed_lengths.values[0]}, but received a length of {len(bs)}.")
else:
- raise bitstring.InterpretError(
- f"'{self.name}' dtypes must have a length in {self.allowed_lengths}, but received a length of {len(bs)}."
- )
+ raise bitstring.InterpretError(f"'{self.name}' dtypes must have a length in {self.allowed_lengths}, but received a length of {len(bs)}.")
return get_fn(bs)
- self.get_fn = allowed_length_checked_get_fn
+ self.get_fn = allowed_length_checked_get_fn # Interpret everything and check the length
else:
- self.get_fn = get_fn
+ self.get_fn = get_fn # Interpret everything
+
+ # Create a reading function from the get_fn.
if not self.variable_length:
if self.allowed_lengths.only_one_value():
-
def read_fn(bs, start):
- return self.get_fn(bs[start:start + self.
- allowed_lengths.values[0]])
+ return self.get_fn(bs[start:start + self.allowed_lengths.values[0]])
else:
-
def read_fn(bs, start, length):
if len(bs) < start + length:
- raise bitstring.ReadError(
- f'Needed a length of at least {length} bits, but only {len(bs) - start} bits were available.'
- )
+ raise bitstring.ReadError(f"Needed a length of at least {length} bits, but only {len(bs) - start} bits were available.")
return self.get_fn(bs[start:start + length])
self.read_fn = read_fn
else:
-
+ # We only find out the length when we read/get.
def length_checked_get_fn(bs):
x, length = get_fn(bs)
if length != len(bs):
@@ -252,47 +317,87 @@ class DtypeDefinition:
self.read_fn = read_fn
self.bitlength2chars_fn = bitlength2chars_fn
- def __repr__(self) ->str:
- s = (
- f"{self.__class__.__name__}(name='{self.name}', description='{self.description}', return_type={self.return_type.__name__}, "
- )
- s += (
- f'is_signed={self.is_signed}, set_fn_needs_length={self.set_fn_needs_length}, allowed_lengths={self.allowed_lengths!s}, multiplier={self.multiplier})'
- )
+ def get_dtype(self, length: Optional[int] = None, scale: Union[None, float, int] = None) -> Dtype:
+ if self.allowed_lengths:
+ if length is None:
+ if self.allowed_lengths.only_one_value():
+ length = self.allowed_lengths.values[0]
+ else:
+ if length not in self.allowed_lengths:
+ if self.allowed_lengths.only_one_value():
+ raise ValueError(f"A length of {length} was supplied for the '{self.name}' dtype, but its only allowed length is {self.allowed_lengths.values[0]}.")
+ else:
+ raise ValueError(f"A length of {length} was supplied for the '{self.name}' dtype which is not one of its possible lengths (must be one of {self.allowed_lengths}).")
+ if length is None:
+ d = Dtype._create(self, None, scale)
+ return d
+ if self.variable_length:
+ raise ValueError(f"A length ({length}) shouldn't be supplied for the variable length dtype '{self.name}'.")
+ d = Dtype._create(self, length, scale)
+ return d
+
+ def __repr__(self) -> str:
+ s = f"{self.__class__.__name__}(name='{self.name}', description='{self.description}', return_type={self.return_type.__name__}, "
+ s += f"is_signed={self.is_signed}, set_fn_needs_length={self.set_fn_needs_length}, allowed_lengths={self.allowed_lengths!s}, multiplier={self.multiplier})"
return s
class Register:
"""A singleton class that holds all the DtypeDefinitions. Not (yet) part of the public interface."""
+
_instance: Optional[Register] = None
names: Dict[str, DtypeDefinition] = {}
- def __new__(cls) ->Register:
+ def __new__(cls) -> Register:
+ # Singleton. Only one Register instance can ever exist.
if cls._instance is None:
cls._instance = super(Register, cls).__new__(cls)
return cls._instance
@classmethod
- def __getitem__(cls, name: str) ->DtypeDefinition:
+ def add_dtype(cls, definition: DtypeDefinition):
+ cls.names[definition.name] = definition
+ if definition.get_fn is not None:
+ setattr(bitstring.bits.Bits, definition.name, property(fget=definition.get_fn, doc=f"The bitstring as {definition.description}. Read only."))
+ if definition.set_fn is not None:
+ setattr(bitstring.bitarray_.BitArray, definition.name, property(fget=definition.get_fn, fset=definition.set_fn, doc=f"The bitstring as {definition.description}. Read and write."))
+
+ @classmethod
+ def add_dtype_alias(cls, name: str, alias: str):
+ cls.names[alias] = cls.names[name]
+ definition = cls.names[alias]
+ if definition.get_fn is not None:
+ setattr(bitstring.bits.Bits, alias, property(fget=definition.get_fn, doc=f"An alias for '{name}'. Read only."))
+ if definition.set_fn is not None:
+ setattr(bitstring.bitarray_.BitArray, alias, property(fget=definition.get_fn, fset=definition.set_fn, doc=f"An alias for '{name}'. Read and write."))
+
+ @classmethod
+ def get_dtype(cls, name: str, length: Optional[int], scale: Union[None, float, int] = None) -> Dtype:
+ try:
+ definition = cls.names[name]
+ except KeyError:
+ raise ValueError(f"Unknown Dtype name '{name}'. Names available: {list(cls.names.keys())}.")
+ else:
+ return definition.get_dtype(length, scale)
+
+ @classmethod
+ def __getitem__(cls, name: str) -> DtypeDefinition:
return cls.names[name]
@classmethod
- def __delitem__(cls, name: str) ->None:
+ def __delitem__(cls, name: str) -> None:
del cls.names[name]
- def __repr__(self) ->str:
- s = [
- f"{'key':<12}:{'name':^12}{'signed':^8}{'set_fn_needs_length':^23}{'allowed_lengths':^16}{'multiplier':^12}{'return_type':<13}"
- ]
+ def __repr__(self) -> str:
+ s = [f"{'key':<12}:{'name':^12}{'signed':^8}{'set_fn_needs_length':^23}{'allowed_lengths':^16}{'multiplier':^12}{'return_type':<13}"]
s.append('-' * 85)
for key in self.names:
m = self.names[key]
allowed = '' if not m.allowed_lengths else m.allowed_lengths
ret = 'None' if m.return_type is None else m.return_type.__name__
- s.append(
- f'{key:<12}:{m.name:>12}{m.is_signed:^8}{m.set_fn_needs_length:^16}{allowed!s:^16}{m.multiplier:^12}{ret:<13} # {m.description}'
- )
+ s.append(f"{key:<12}:{m.name:>12}{m.is_signed:^8}{m.set_fn_needs_length:^16}{allowed!s:^16}{m.multiplier:^12}{ret:<13} # {m.description}")
return '\n'.join(s)
+# Create the Register singleton
dtype_register = Register()
diff --git a/bitstring/exceptions.py b/bitstring/exceptions.py
index 1cabe0f..374d393 100644
--- a/bitstring/exceptions.py
+++ b/bitstring/exceptions.py
@@ -1,7 +1,8 @@
+
class Error(Exception):
"""Base class for errors in the bitstring module."""
- def __init__(self, *params: object) ->None:
+ def __init__(self, *params: object) -> None:
self.msg = params[0] if params else ''
self.params = params[1:]
diff --git a/bitstring/fp8.py b/bitstring/fp8.py
index 4cf9431..575dbbb 100644
--- a/bitstring/fp8.py
+++ b/bitstring/fp8.py
@@ -3,6 +3,7 @@ The 8-bit float formats used here are from a proposal supported by Graphcore, AM
See https://arxiv.org/abs/2206.02915
"""
+
import struct
import zlib
import array
@@ -17,20 +18,80 @@ class Binary8Format:
def __init__(self, exp_bits: int, bias: int):
self.exp_bits = exp_bits
self.bias = bias
- self.pos_clamp_value = 127
- self.neg_clamp_value = 255
+ self.pos_clamp_value = 0b01111111
+ self.neg_clamp_value = 0b11111111
def __str__(self):
- return f'Binary8Format(exp_bits={self.exp_bits}, bias={self.bias})'
+ return f"Binary8Format(exp_bits={self.exp_bits}, bias={self.bias})"
+
+ def decompress_luts(self):
+ binary8_to_float_compressed, float16_to_binary8_compressed = binary8_luts_compressed[(self.exp_bits, self.bias)]
+ self.lut_float16_to_binary8 = zlib.decompress(float16_to_binary8_compressed)
+ dec = zlib.decompress(binary8_to_float_compressed)
+ self.lut_binary8_to_float = struct.unpack(f'<{len(dec) // 4}f', dec)
+
+ def create_luts(self):
+ self.lut_binary8_to_float = self.createLUT_for_binary8_to_float()
+ self.lut_float16_to_binary8 = self.createLUT_for_float16_to_binary8()
- def float_to_int8(self, f: float) ->int:
+ def float_to_int8(self, f: float) -> int:
"""Given a Python float convert to the best float8 (expressed as an integer in 0-255 range)."""
- pass
+ # First convert the float to a float16, then a 16 bit uint
+ try:
+ b = struct.pack('>e', f)
+ except (OverflowError, struct.error):
+ # Return the largest representable positive or negative value
+ return self.pos_clamp_value if f > 0 else self.neg_clamp_value
+ f16_int = int.from_bytes(b, byteorder='big')
+ # Then use this as an index to our large LUT
+ return self.lut_float16_to_binary8[f16_int]
+
+ def createLUT_for_float16_to_binary8(self) -> bytes:
+ # Used to create the LUT that was compressed and stored for the fp8 code
+ import gfloat
+ fi = gfloat.formats.format_info_p3109(8 - self.exp_bits)
+ fp16_to_fp8 = bytearray(1 << 16)
+ for i in range(1 << 16):
+ b = struct.pack('>H', i)
+ f, = struct.unpack('>e', b)
+ fp = gfloat.round_float(fi, f)
+ if math.isnan(fp):
+ fp8_i = 0b10000000
+ else:
+ fp8_i = self.lut_binary8_to_float.index(fp)
+ fp16_to_fp8[i] = fp8_i
+ return bytes(fp16_to_fp8)
def createLUT_for_binary8_to_float(self):
"""Create a LUT to convert an int in range 0-255 representing a float8 into a Python float"""
- pass
+ i2f = []
+ for i in range(256):
+ b = bitarray.util.int2ba(i, length=8, endian='big', signed=False)
+ sign = b[0]
+ exponent = bitarray.util.ba2int(b[1:1 + self.exp_bits])
+ significand = b[1 + self.exp_bits:]
+ if exponent == 0:
+ significand = bitarray.bitarray('0') + significand
+ exponent = -self.bias + 1
+ else:
+ significand = bitarray.bitarray('1') + significand
+ exponent -= self.bias
+ f = float(bitarray.util.ba2int(significand)) / (2.0 ** (7 - self.exp_bits))
+ f *= 2 ** exponent
+ i2f.append(f if not sign else -f)
+ # One special case for minus zero
+ i2f[0b10000000] = float('nan')
+ # and for plus and minus infinity
+ i2f[0b01111111] = float('inf')
+ i2f[0b11111111] = float('-inf')
+ return array.array('f', i2f)
+# We create the 1.5.2 and 1.4.3 formats.
p4binary_fmt = Binary8Format(exp_bits=4, bias=8)
p3binary_fmt = Binary8Format(exp_bits=5, bias=16)
+
+
+def decompress_luts():
+ p4binary_fmt.decompress_luts()
+ p3binary_fmt.decompress_luts()
diff --git a/bitstring/luts.py b/bitstring/luts.py
index 07452b3..f4ca298 100644
--- a/bitstring/luts.py
+++ b/bitstring/luts.py
@@ -1,38 +1,245 @@
-mxfp_luts_compressed = {(2, 1, 1, 'saturate'): (
- b'x\x01\x1d\xc9\xc1\r\xc00\x00\xc2@o\xd6\x8c\xc6f\xf5h\xb1\x828\xf1\x00^>X\x0c\xa7f1,\x7f\x13\x83\xfdY\xf4\x027\xf1\x0c\xfb'
- ,
- b'x\x01\xed\xdd\t\r\xc00\x10\x03\xc1\xf4\xff[\xfehS\x1a\'y\x96\x81\x87\x80[\x13\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02e\x05\x06\x11 @\x80@\xa2\xc0(\x02\xc9\x02\x93\x92\x05fe\x0b,J\x16X\x95-\xb0\x89\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x14\x14\xe8"@ U`\x17\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02u\x05\x0e\x11 @\x80@\xa2\xc0)\x02\xc9\x02\x97\x92\x05ne\x0b<J\x16x\x95-\xf0\x89\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x14\x14H=>\xb7\x9b\x00\x81\xfe\x03\\M8\xf2'
- ), (2, 3, 1, 'saturate'): (
- b'x\x01\x1d\xcdQ\r\xc30\x10\x03\xd0\x830\x08\x85P\x04[ \x14B \x0cB \x04B \x14\xc1v\x10\x02!\x10\x02aO\xb3\xf4\xe4?;\xe2\x9fgD#\x89W\xc4A\xa1\xd2\xe8\x0cn\x92\xc9b\x13%\xe2\xc1\xc1I\xe1\xa2\xf2\xa6\xd1\x19\xdc$\x93\xc5&\x1a\x1fE\x12_[\x14*\x8d\xce\xe0&\x99,6\x91\xfe98)\\T\xde4:\x83\x9bd\xb2\xd8\xf9\x03~S=\xdd'
- ,
- b'x\x01\xed\xdd\x85qB\x01\x14DQ\x12\x88\x11#\xeeF\x84\xb8\xbb\xf6_\x15\xa9\x82\xd9\xf9\xf3\xce\xed`O\x03\xdbj\x89\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x81\xc6\x08\x8c\x89\x00\x01\x02\x04*\n\x8c\x8b@e\x81\xb6*\x0bt\x94\x15\x98\xa8\xddd\xf5\xa6j7\xdd\xf4f\xb2u\xd3\xcdf\x9bK7\x9fm!\xddb\xb6^\xba\xa5l\xcb\xe9V\xb2\xad\xa6[\xcb\xb6\x9en#\xdbf\xba\xadl\xdb\xe9v\xb2\xed\xa6\xdb\xcb\xb6\x9f\xee@\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10\x18\x91\xc0P\x04\x08T\x158\x14\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\xcd\x118\x12\x01\x02\x04\x08T\x14\xe8\x8b@e\x81cU\x168QV\xe0\xb4vg\xd5\x1b\xd4\xee\xbc\xe9]d\xbbLw\x95\xed:\xddM\xb6\xdbtw\xd9\xee\xd3=d{L\xf7\x94\xed9\xddK\xb6\xd7to\xd9\xde\xd3}d\xfbL\xf7\x95\xed;\xddO\xb6\xdft\x7f"@\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\xc0\x88\x04\xaa\x1e\x9f\xdbM\x80\xc0\xf0\x1f\x9f\x0cK\xfb'
- ), (3, 2, 3, 'saturate'): (
- b'x\x01\x15\xcbA\x15BQ\x08\x05@\xa2XD\xc5&/\nQ(\xa2\xf2\x9b\x10\xc5\x91\xc3\xc0\xe2B\xc4\xbf\xean<"\x92\xa2\x19\x96xF\xdcH\x0eE3,\x91r\x92C\xd1\x0cK\xbc\xe4$\x87\xa2\x19\x96(\xfd\xb6?n(\x9aa\x89\xaf\x7f\x92C\xd1\x0cK\x8c\x9c\xe4P4\xc3\x12\x97\x9c\xe4P4\xc3^?\xc7\x8a;c'
- ,
- b'x\x01\xed\xdd\xd7U\x15\x00\x00\x04Q$\x89HV@r\x90 Q\xb2\x92\xfb\xaf\xea\xd1\x02\x7f\x1c\xce\xdc\xe9`o\x03;4$\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\xe0\xd3\t|\x11\x01\x02\x04\x08\x14\x05\x86E\xa0,0\xa2\xb2\xc0\xa8>V`\xac\xddx\xbd\xaf\xed&\xea}k7Y\xef{\xbb\xa9z\xd3\xedf\xea\xcd\xb6\x9b\xab7\xdfn\xa1\xde\x8fv?\xeb-\xb6[\xaa\xb7\xdc\xeeW\xbd\x95v\xab\xf5\xd6\xda\xad\xd7\xdbh\xb7YoK\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 \xf0.\x81\x81\x08\x10\xa8\nl\x8b\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10\xf8|\x02;"@\x80\x00\x81\xa2\xc0\xae\x08\x94\x05~\xab,\xb0\xa7\x8f\x15\xd8owP\xef\xb0\xdd\x9fzG\xed\x8e\xeb\x9d\xb4;\xadw\xd6\xee\xbc\xde\xdfv\x17\xf5.\xdb]\xd5\xbbnwS\xef\xb6\xdd]\xbd\x7f\xed\xfe\xd7\xbbo\xf7P\xef\xb1\xddS\xbd\xe7v/\xf5^E\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\xef\x12\xa8\x1e\x9f\xdbM\x80\xc0\xe0\r\xd9\xf2;{'
- ), (4, 3, 7, 'saturate'): (
- b'x\x01\x1d\xcd[\xb5\x90!\x10\x80Q"\x18\x81\x08\xbc{C-@\x04"\xd0\xe0\x10\x81\x08D\xe0\xdd\x1bj\x01"\x10\xe1o\xa0\xfb\xc8Z{\xcd\xbc|C\x08\xff\xdf\xdb\x10:\x9b\xf0.\x84H\xa6\xd2\x19L\x16\x9b\xc3\xe5!\xbc\x0f\xe1\r\x91D\xa6Pit\x06\x93\xc5\xe6py\x08\x1f\xf4D\x12\x99B\xa5\xd1\x19L\x16\x9b\xc3\xe5!|\xd4\x13Id\n\x95Fg0Yl\x0e\x97\x87\x90\xf5D\x12\x99B\xa5\xd1\x19L\x16\x9b\xc3\xe5!|\xd2\x13Id\n\x95Fg0Yl\x0e\x97\x87\xf0YO$\x91)T\x1a\x9d\xc1d\xb19\\\x1e\xc2\x17=\x91D\xa6Pit\x06\x93\xc5\xe6p_\xf7\x17}\xe7\xab\xc1&|s\x8bL\xa53\x98,6\x87\xcbC\xf8\xee\x7f"\x89L\xa1\xd2\xe8\x0c&\x8b\xcd\xe1\xf2\x10~\xe8\x89$2\x85J\xa33\x98,6\x87\xcbC\xf8\xa9\'\x92\xc8\x14*\x8d\xce`\xb2\xd8\x1c.\x0fa\xeb\x89$2\x85J\xa33\x98,6\x87\xcbC\xf8\xa5\'\x92\xc8\x14*\x8d\xce`\xb2\xd8\x1c.\x0f\xe1\xb7\x9eH"S\xa84:\x83\xc9bs\xb8<\x84?z"\x89L\xa1\xd2\xe8\x0c&\x8b\xcd\xe1\xbe\xee\x7f\xff\x01!\xba\xf7\x9b'
- ,
- b'x\x01\xed\xdd\xd5\x96\x90\x05\x00E\xe1\x01\xe9\x90n\x90\x96F\xbaE\xa4S@\xba\x15\xa4\xbbKX H*\xdd)\xa9\x12Jw*H\x08Jww\x87\xd25\\\xccC\xec5\xf3\xef\xef\r\xce~\x81\x13\x12"\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX \xc2\x16\x88$\x0bX\xc0\x02\x16\x08b\x81\xc8\xb2@\x90\x0b|\xa0 \x17\x88"\xb6@\xd4`\x8b\x16t\xd1\x83-Fx\x17\x93\x15\x8b\x16\x9b\x15\x87\x16\x97\xf5!-\x1e+>-\x01+!-\x11+1-\t+)-\x19+9-\x05+%-\x15+5-\r\xeb#ZZV:ZzV\x06ZFV&Zf\xd6\xc7\xb4,\xac\xac\xb4l\xac\xec\xb4\x1c\xac\x9c\xb4\\\xac\xdc\xb4OXyhyY\xf9h\xf9Y\x05h\x05Y\x85h\x85YEhEY\xc5h\xc5Y%h\x9f\xb2J\xd2>c\x95\xa2}\xce*M+\xc3*K+\xc7*O\xab\xc0\xaaH\xab\xc4\xaaL\xab\xc2\xaaJ\xab\xc6\xfa\x82V\x9dU\x83V\x93\xf5%\xad\x16\xab6\xad\x0e\xab.\xad\x1e\xab>\xad\x01\xab!\xad\x11\xab1\xad\t\xab)\xad\x19\xeb+\xda\xd7\xac\xe6\xb4\x16\xacoh-Y\xadh\xadYmhmY\xedh\xedY\x1dh\x1dY\x9dh\x9dY]h]Y\xddh\xddY=h=Y\xbdh\xbdY}h}Y\xfdh\xfdY\xdf\xd2\x06\xb0\x06\xca\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05\xc2\n\x84\xca\x02\x16\x08j\x81A\xb2\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05"n\x81\xc1\xb2\x80\x05,`\x81 \x16\xf8N\x16\x08r\x81!\nr\x81\xa1b\x0b|\x1fl\xc3\x82nx\xb0\x8d\x08\xefF\xb2F\xd1F\xb3~\xa0\xfd\xc8\x1aC\x1b\xcb\x1aG\x1b\xcf\x9a@\x9b\xc8\x9aD\x9b\xcc\x9aB\x9b\xca\x9aF\x9b\xce\x9aA\x9b\xc9\x9aE\x9b\xcd\x9aC\x9b\xcb\xfa\x896\x8f5\x9f\xb6\x80\xb5\x90\xb6\x88\xb5\x98\xf63\xeb\x17\xda\xaf\xac%\xb4\xa5\xace\xb4\xe5\xac\xdfh\xbf\xb3V\xd0V\xb2V\xd1V\xb3\xd6\xd0\xd6\xb2\xd6\xd1\xd6\xb36\xd06\xb26\xd16\xb3\xb6\xd0\xb6\xb2\xb6\xd1\xb6\xb3v\xd0v\xb2\xfe\xa0\xfd\xc9\xdaE\xdb\xcd\xfa\x8b\xb6\x87\xb5\x97\xb6\x8f\xb5\x9f\xf67\xeb\x00\xed \xeb\x1f\xda\xbf\xacC\xb4\xc3\xac#\xb4\xa3\xacc\xb4\xe3\xac\x13\xb4\x93\xacS\xb4\xd3\xac3\xb4\xb3\xacs\xb4\xf3\xac\x0b\xb4\x8b\xacK\xb4\xcb\xac+\xb4\xab\xack\xb4\xeb\xac\x1b\xb4\x9b\xac[\xb4\xdb\xac;\xb4\xbb\xac{\xb4\xfb\xac\x07\xb4\x87\xacG\xb4\xc7\xac\xffh\xff\xb3\x9e\xd0\x9e\xb2\x9e\xd1\x9e\xb3^\xd0^\xb2^\xd1^\xb3\xde\xd0\xde\xb2\xde\xc9\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05\xc2\n\x04\xf5\xf8\xdc\xdd\x16\xb0@\xe8{\t?\xc8\x90'
- ), (5, 2, 15, 'saturate'): (
- b"x\x01M\xcb\xd9\x11\x10T\x0c@\xd1\x94B\x17 \x08\x06Dh#\x9d\x90R\xd2\x05\xca\x1a\x17\xa0\x8dt\x02\xc7\x1f\xc7\xcc\x9c\xf7\xf1n\x12\xf1\xef\xf4C\xcf\xa3\x88\xa4\x19\x96#~\x8ax@R4\xc3r\xc4c\x9d\xa4h\x86\xe5\x88':I\xd1\x0c\xcb\x11?\xeb$E3,G<\xd5I\x8afX\x8ex\xa6\x93\x14\xcd\xb0\x1c\xf1\x8bNR4\xc3rD\xea$E3,G<\xd7I\x8afX\x8ex\xa1\x93\x14\xcd\xb0\x1c\xf1\xabNR4\xc3r\xc4K\x9d\xa4h\x86\xe5\x88\xdft\x92\xa2\x19\x96#^\xe9$E3,G\xbc\xd6I\x8a~\xa3\xfdO\xb4\xbf\xb7\xf6~\xb7C3,G\xfc\xe1\x9e\xa4h\x86\xe5\x88w:I\xd1\x0c\xcb\x11\xefu\x92\xa2\x19\x96#>\xe8$E3,G|\xd4I\x8afX\x8e\xf8\xa4\x93\x14\xcd\xb0\x1c\xf1Y')\x9aa9bu\x92\xa2\x19\x96#\xfe\xd4I\x8afX\x8e\xf8K')\x9aa9\xe2o\x9d\xa4h\x86\xe5\x88\x7ft\x92\xa2\x19\x96#\xbe\xe8$E3,G|\xd5I\x8afX\x8e\xf8\xa6\x93\x14\xfd]\xfb\xcf\x0f\xd2\x15\xf0\xcf"
- ,
- b"x\x01\xed\xddC\xa2\x1c\x00\x00D\xc1\xd8\xb6m\xdb\xb6m\xdb66\xb1m\xdb\xb6m\xdb\xb6m,\x92c\xfcEM\xdd\xa0\xdf\x05:X\xb0 \x16\xdc\x16B\x17\xd2\x16J\x17\xda\x16F\x17\xd6\x16N\x17\xde\x16A\x17\xd1\x16I\x17\xd9\x16E\x17\xd5\x16M\x17\xdd\x16C\x17\xd3\x16K\x17\xdb\x16G\x17\xd7\x16O\x17\xdf\x96@\x97\xd0\x96H\x97\xd8\x96D\x97\xd4\x96L\x97\xdc\x96B\x97\xd2\x96J\x97\xda\x96F\x97\xd6\x96N\x97\xde\x96A\x97\xd1\x96I\x97\xd9\x96E\x97\xd5\x96M\x97\xdd\x96C\x97\xd3\x96K\x97\xdb\x96G\x97\xd7\x96O\x97\xdfV@W\xd0VHW\xd8VDW\xd4VLW\xdcVBW\xd2VJW\xdaVFW\xd6VNW\xdeVAW\xd1VIW\xd9VEW\xd5VMW\xddVCW\xd3VKW\xdbVGW\xd7VOW\xdf\xd6@\xd7\xd0\xd6H\xd7\xd8\xd6D\xd7\xd4\xd6L\xd7\xdc\xd6B\xd7\xd2\xd6J\xd7\xda\xd6F\xd7\xd6\xd6N\xd7\xde\xd6A\xd7\xd1\xd6I\xd7\xd9\xd6E\xd7\xd5\xd6M\xd7\xdd\xd6C\xd7\xd3\xd6K\xd7\xdb\xd6G\xd7\xd7\xd6O\xd7\xdf6@70 H\x0b\xfc\x0b\x08\x14\x08\x14P\x0b\x0c\nj\x83mCtCm\xc3t\xc3m#t#m\xa3t\xa3mctcm\xe3t\xe3m\x13t\x13m\x93t\x93mStSm\xd3t\xd3m3t3m\xb3t\xb3mstsm\xf3t\xf3m\x0bt\x0bm\x8bt\x8bmKtKm\xcbt\xcbm+t+m\xabt\xabmktkm\xebt\xebm\x1bt\x1bm\x9bt\x9bm[t[m\xdbt\xdbm;t;m\xbbt\xbbm{t{m\xfbt\xfbm\x07t\x07m\x87t\x87mGtGm\xc7t\xc7m't'm\xa7t\xa7mgtgm\xe7t\xe7m\x17t\x17m\x97t\x97mWtWm\xd7t\xd7m7t7m\xb7t\xb7mwtwm\xf7t\xf7m\x0ft\x0fm\x8ft\x8fmOtOm\xcft\xcfm/t/m\xaft\xafmotom\xeft\xefm\x1ft\x1fm\x9ft\x9fm_t_m\xdft\xdfm?t?m\xbft\xbfm\x7ft\x7f\x03\x82\xb4\x80z|\x1e\xd8\x1d(\x10(\xf0\xef?\xe6\xfc\r\x9b"
- ), (4, 3, 7, 'overflow'): (
- b'x\x01\x1d\xcd[\xb5\x90!\x10\x80Q"\x18\x81\x08\xbc{C-@\x04"\xd0\xe0\x10\x81\x08D\xe0\xdd\x1bj\x01"\x10\xe1o\xa0\xfb\xc8Z{\xcd\xbc|C\x08\xff\xdf\xdb\x10:\x9b\xf0.\x84H\xa6\xd2\x19L\x16\x9b\xc3\xe5!\xbc\x0f\xe1\r\x91D\xa6Pit\x06\x93\xc5\xe6py\x08\x1f\xf4D\x12\x99B\xa5\xd1\x19L\x16\x9b\xc3\xe5!|\xd4\x13Id\n\x95Fg0Yl\x0e\x97\x87\x90\xf5D\x12\x99B\xa5\xd1\x19L\x16\x9b\xc3\xe5!|\xd2\x13Id\n\x95Fg0Yl\x0e\x97\x87\xf0YO$\x91)T\x1a\x9d\xc1d\xb19\\\x1e\xc2\x17=\x91D\xa6Pit\x06\x93\xc5\xe6p_\xf7\x17}\xe7\xab\xc1&|s\x8bL\xa53\x98,6\x87\xcbC\xf8\xee\x7f"\x89L\xa1\xd2\xe8\x0c&\x8b\xcd\xe1\xf2\x10~\xe8\x89$2\x85J\xa33\x98,6\x87\xcbC\xf8\xa9\'\x92\xc8\x14*\x8d\xce`\xb2\xd8\x1c.\x0fa\xeb\x89$2\x85J\xa33\x98,6\x87\xcbC\xf8\xa5\'\x92\xc8\x14*\x8d\xce`\xb2\xd8\x1c.\x0f\xe1\xb7\x9eH"S\xa84:\x83\xc9bs\xb8<\x84?z"\x89L\xa1\xd2\xe8\x0c&\x8b\xcd\xe1\xbe\xee\x7f\xff\x01!\xba\xf7\x9b'
- ,
- b"x\x01\xed\xdc\x05\xb2\x90\x05\x00E\xe1\x87\x80\xa4t\x83tw\x97Jw\x0bHKw\x83\x80\x92Cww7\x92\xd2\x1dRJww\x97\xa4\x92J\xba\x8c3\x8f\xff|\x1b\xb83g\x017$D\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\xc0g[ \x8c,`\x01\x0bX \x88\x05\xbe\x90\x05\x82\\ \xac\x82\\ \x9c\xd8\x02\xe1\x83\xed\xcb\xa0\x8b\x10l\x11C\xbbH\xac\xc8\xb4(\xac\xa8\xb4\xafX\xd1h\xd1Y1h1Y\xb1h\xb1YqhqY\xf1h\xf1Y\th\tY\x89h\x89YIh_\xb3\x92\xd2\x92\xb1\x92\xd3R\xb0R\xd2R\xb1R\xd3\xd2\xb0\xd2\xd2\xd2\xb1\xd2\xd32\xb02\xd22\xb12\xd3\xb2\xb0\xb2\xd2\xb2\xb1\xb2\xd3r\xb0r\xd2r\xb1r\xd3\xf2\xb0\xf2\xd2\xf2\xb1\xf2\xd3\n\xb0\n\xd2\xbea}K\xfb\x8eU\x88V\x98U\x84V\x94U\x8cV\x9cU\x82V\x92U\x8aV\x9aU\x86V\x96U\x8eV\x9eU\x81V\x91U\x89V\x99U\x85\xf6=\xab*\xad\x1a\xab:\xed\x07V\rZMV-ZmV\x1dZ]V=\xda\x8f\xac\xfa\xb4\x06\xac\x86\xb4F\xac\xc6\xb4&\xac\xa6\xb4f\xac\xe6\xb4\x16\xac\x96\xb4V\xac\xd6\xb46\xac\xb6\xb4v\xac\xf6\xb4\x0e\xac\x8e\xb4N\xac\xce\xb4\x9fX]h]Y\xddh?\xb3~\xa1ug\xf5\xa0\xf5d\xf5\xa2\xf5f\xf5\xa1}\x92\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16`\x0b\xf4\x95\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,\xf0\xf9\x16\xe8'\x0bX\xc0\x02\x16\x08b\x81\xfe\xb2@\x90\x0b\x0cP\x90\x0b\x0c\x14[`P\xb0\r\x0e\xba!\xc164\xb4\x1b\xc6\x1aN\x1b\xc1\x1aI\x1b\xc5\x1aM\x1b\xc3\x1aK\x1b\xc7\x1aO\x9b\xc0\x9aH\x9b\xc4\x9aL\x9b\xc2\x9aJ\x9b\xc6\x9aN\x9b\xc1\x9aI\x9b\xc5\x9aM\x9b\xc3\x9aK\x9b\xc7\x9aO[\xc0ZH[\xc4\xfa\x95\xb6\x98\xb5\x84\xb6\x94\xb5\x8c\xb6\x9c\xb5\x82\xf6\x1bk%m\x15k5m\rk-m\x1dk=m\x03k#m\x13k3m\x0bk+m\x1bk;m\x07\xebw\xdaN\xd6.\xdan\xd6\x1e\xda^\xd6\x1f\xb4?Y\xfbh\xfbY\x07h\x07Y\x87h\x87YGhGY\xc7h\xc7Y'h'Y\xa7h\xa7YghgY\xe7h\xe7Y\x17h\x17Y\x97h\x97YWhWY\xd7h\xd7Y7h7Y\xb7h\xb7YwhwY\xf7h\xf7Y\x0fh\x7f\xb1\x1e\xd2\x1e\xb1\x1e\xd3\x9e\xb0\x9e\xd2\x9e\xb1\xfe\xa6\xfd\xc3zN{\xc1zI{\xc5zM{\xc3\xfa\x97\xf6\x1f\xeb-\xed\x1d\xeb=\xed\x03\xeb#\x8d\xbd\xefw\xdd\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0b|\xfa\x1f\xb2\xf6b\xf1"
- ), (5, 2, 15, 'overflow'): (
- b"x\x01M\xcb\xd9\x11\x10T\x0c@\xd1\x94B\x17 \x08\x06Dh#\x9d\x90R\xd2\x05\xca\x1a\x17\xa0\x8dt\x02\xc7\x1f\xc7\xcc\x9c\xf7\xf1n\x12\xf1\xef\xf4C\xcf\xa3\x88\xa4\x19\x96#~\x8ax@R4\xc3r\xc4c\x9d\xa4h\x86\xe5\x88':I\xd1\x0c\xcb\x11?\xeb$E3,G<\xd5I\x8afX\x8ex\xa6\x93\x14\xcd\xb0\x1c\xf1\x8bNR4\xc3rD\xea$E3,G<\xd7I\x8afX\x8ex\xa1\x93\x14\xcd\xb0\x1c\xf1\xabNR4\xc3r\xc4K\x9d\xa4h\x86\xe5\x88\xdft\x92\xa2\x19\x96#^\xe9$E3,G\xbc\xd6I\x8a~\xa3\xfdO\xb4\xbf\xb7\xf6~\xb7C3,G\xfc\xe1\x9e\xa4h\x86\xe5\x88w:I\xd1\x0c\xcb\x11\xefu\x92\xa2\x19\x96#>\xe8$E3,G|\xd4I\x8afX\x8e\xf8\xa4\x93\x14\xcd\xb0\x1c\xf1Y')\x9aa9bu\x92\xa2\x19\x96#\xfe\xd4I\x8afX\x8e\xf8K')\x9aa9\xe2o\x9d\xa4h\x86\xe5\x88\x7ft\x92\xa2\x19\x96#\xbe\xe8$E3,G|\xd5I\x8afX\x8e\xf8\xa6\x93\x14\xfd]\xfb\xcf\x0f\xd2\x15\xf0\xcf"
- ,
- b"x\x01\xed\xddS\xb6\x1e\x06\x00\x85\xd1\x1b\xdb\xb6\x1b\xdb\xb6\x9b\x06\x8dm\xdbz\x88m\xabI\x1b\xa3\xb1m\xdb\xb6mg\xad\x9ba\xdc\x87\xfd\xef\x19\x9co\x02'((\x84\x85\xb2\x85\xd6\x85\xb1\x85\xd5\x85\xb3\x85\xd7E\xb0E\xd4E\xb2E\xd6E\xb1E\xd5E\xb3E\xd7\xc5\xb0\xc5\xd4\xc5\xb2\xc5\xd6\xc5\xb1\xc5\xd5\xc5\xb3\xc5\xd7%\xb0%\xd4%\xb2%\xd6%\xb1%\xd5%\xb3%\xd7\xa5\xb0\xa5\xd4\xa5\xb2\xa5\xd6\xa5\xb1\xa5\xd5\xa5\xb3\xfd\xa1Ko\xcb\xa0\xcbh\xcb\xa4\xcbl\xcb\xa2\xcbj\xcb\xa6\xcbn\xcb\xa1\xcbi\xcb\xa5\xcbm\xcb\xa3\xcbk\xcb\xa7\xcbo+\xa0+h+\xa4+l+\xa2+j+\xa6+n+\xa1+i+\xa5+m+\xa3+k+\xa7+o\xab\xa0\xabh\xab\xa4\xabl\xfbSW\xc5\xf6\x97\xae\xaa\xad\x9a\xae\xba\xad\x86\xeeo[M]-[m]\x1d[]]=[}]\x03[C]#[c]\x13[S]3[s]\x0b[K]+[k]\x1b[[];[{]\x07[G]'[g]\x17[W]7[w]\x0f[O]/[o]\x1f[_]?[\x7f\xdd\x00\xdb\xc0\x90\x16\x1c\x10(\x10(\xa0\x16\x18\x14\xd2\x06\xdb\x86\xe8\x86\xda\x86\xe9\x86\xdbF\xe8F\xdaF\xe9F\xdb\xc6\xe8\xc6\xda\xc6\xe9\xc6\xdb&\xe8&\xda&\xe9&\xdb\xa6\xe8\xa6\xda\xa6\xe9\xa6\xdbf\xe8f\xdaf\xe9f\xdb\xe6\xe8\xe6\xda\xe6\xe9\xfe\xb1\xcd\xd7-\xb0\xfd\xab\xfb\xcf\xb6P\xb7\xc8\xb6X\xb7\xc4\xb6T\xb7\xcc\xb6\\\xb7\xc2\xb6R\xb7\xca\xf6\xbfn\xb5m\x8dn\xadm\x9dn\xbdm\x83n\xa3m\x93n\xb3m\x8bn\xabm\x9bn\xbbm\x87n\xa7m\x97n\xb7m\x8fn\xafm\x9fn\xbf\xed\x80\xee\xa0\xed\x90\xee\xb0\xed\x88\xee\xa8\xed\x98\xee\xb8\xed\x84\xee\xa4\xed\x94\xee\xb4\xed\x8c\xee\xac\xed\x9c\xee\xbc\xed\x82\xee\xa2\xed\x92\xee\xb2\xed\x8a\xee\xaa\xed\x9a\xee\xba\xed\x86\xee\xa6\xed\x96\xee\xb6\xed\x8e\xee\xae\xed\x9e\xee\xbe\xed\x81\xee\xa1\xed\x91\xee\xb1\xed\x89\xee\xa9\xed\x99\xee\xb9\xed\x85\xee\xa5\xed\x95\xee\xb5\xed\x8d\xee\xad\xed\x9d\xee\xbd\xed\x83\xee\xa3\xed\x93\xee\xb3\xed\x8b\xee\xab\xed\x9b\xee\xbb\xed\x87\xee\xa7\xedWHS\x8f\xcf\x03\xbb\x03\x05\x02\x05\x82\x7f\x03\xb3\x87\x0e\x9d"
- )}
-binary8_luts_compressed = {(4, 8): (
- b'x\x01\x15\xcb[\xd5P!\x10\x80Q"\x18\x81\x08<{E-@\x04"\xd0@"\x10\x81\x08<{E\xff\x02\'\x02\x11h\xa0\xdbYk\xcf\xcb\xcc\x17\xc2\xff\xe9\xaf\xad7!d:\x93\xcd!\xbc\r\xe1\x15\x91D\xa6Pit\x06\x93\xc5\xe6\xe1p\t\xef\xf4D\x12\x99B\xa5\xd1\x19L\x16\x9b\x87\xc3%\xbc\xd7\x13Id\n\x95Fg0Yl\x1e\x0e\x97\xf0AO$\x91)T\x1a\x9d\xc1d\xb1y8\\B\xd6\x13Id\n\x95Fg0Yl\x1e\x0e\x97\xf0QO$\x91)T\x1a\x9d\xc1d\xb1y8\\\xc2\'=\x91D\xa6Pit\x06\x93\xc5\xe6\xe1p\t\x9f\xf5D\x12\x99B\xa5\x7f\xf1O\xff\xea\xef\x9b\x1b\x9d\xc9\xe6\x10\xbe\xeb\x89$2\x85J\xa33\x98,6\x0f\x87K\xf8\xa1\'\x92\xc8\x14*\x8d\xce`\xb2\xd8<\x1c.\xe1\xa7\x9eH"S\xa84:\x83\xc9b\xf3p\xb8\x84_z"\x89L\xa1\xd2\xe8\x0c&\x8b\xcd\xc3\xe1\x12\xb6\x9eH"S\xa84:\x83\xc9b\xf3p\xb8\x84\xdfz"\x89L\xa1\xd2\xe8\x0c&\x8b\xcd\xc3\xe1\x12\xfe\xe8\x89$2\x85J\xa33\x98,6\x0f\x87Kx\xd1\x13Id\n\x95\xfe\xf7\x1f[)\xf3`'
- ,
- b"x\x01\xed\xdd\x05\xba\x96\x05\x00\x05\xe1\x9f\xee\x06\xe9FZA\xa4\xbb\xbb;\xa4SB\xba\xeb\xd2\xdd\x8dt\x97\x92J(\xa14\xa2\x84\x92\x8a\xa4\x82\xd2\x1d\x12.c\x9e\xcb7\xef\x0e\xcel\xe0\x84B\xb2\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xe0\xbd/\x10A\x16\xb0\x80\x05,\x10\xc4\x02\x11e\x81 \x17\x88\xa4 \x17\x88,\xb6@\x94`\x8b\x1at\xd1\x82-zx\x17\x83\x15\x93\x16\x8b\x15\x9b\x16\x87\x15\x97\x16\x8f\x15\x9f\x96\x80\x95\x90\x96\x88\x95\x98\x96\x84\xf5\x01-)+\x19-9+\x05-%+\x15-5+\r--+\x1d-=+\x03-#+\x13\xedCVfZ\x16VVZ6VvZ\x0eVN\xdaG\xac\x8fi\xb9X\xb9i\x9f\xb0\xf2\xd0>e\xe5\xa5\xe5c\xe5\xa7\x15`\x15\xa4\x15b\x15\xa6\x15a\x15\xa5\x15c\x15\xa7\x95`\x95\xa4\x95b\x95\xa6\x95a\x95\xa5\x95c\x95\xa7U`U\xa4UbU\xa6UaU\xa5UcU\xa7\xd5`\xd5\xa4\xd5b\xd5\xa6\xd5a\xd5\xa5\xd5c\xd5\xa75`5\xa45b5\xa65a}Fk\xcajFk\xcejAk\xc9jEk\xcdjCk\xcbjGk\xcf\xea@\xfb\x9c\xd5\x91\xd6\x89\xd5\x99\xd6\x85\xf5\x05\xad+\xab\x1b\xad;\xab\x07\xad'\xab\x17\xad7\xab\x0f\xad/\xab\x1f\xad?k\x00m k\x10m0k\x08m\xa8,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x00\\ L\x16\xb0@P\x0b\xbc\xf7\xff\x86\x0e\xb4\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX \x14\x1a&\x0bX\xc0\x02\x16\x08b\x81\xe1\xb2@\x90\x0b\x8cP\x90\x0b\x8c\x14[`T\xb0\x8d\x0e\xba1\xc166\xbc\x1b\xc7\x1aO\x9b\xc0\x9aH\x9b\xc4\x9aL\x9b\xc2\x9aJ\x9b\xc6\x9aN\x9b\xc1\x9aI\x9b\xc5\x9aM\x9b\xc3\x9aK\x9b\xc7\x9aO\xfb\x92\xb5\x80\xb6\x90\xb5\x88\xb6\x98\xb5\x84\xb6\x94\xb5\x8c\xb6\x9c\xb5\x82\xb6\x92\xb5\x8a\xb6\x9a\xb5\x86\xb6\x96\xb5\x8e\xb6\x9e\xb5\x81\xf6\x15\xebk\xdaF\xd6&\xdaf\xd6\x16\xdaV\xd66\xda7\xacoi\xdbY;h;Y\xbbh\xdf\xb1\xbe\xa7\xedf\xed\xa1\xede\xed\xa3\xfd\xc0\xfa\x91\xb6\x9fu\x80v\x90u\x88v\x98u\x84v\x94u\x8c\xf6\x13\xeb8\xedg\xd6/\xb4\x13\xac\x93\xb4S\xac\xd3\xb4_Y\xbf\xd1\xce\xb0\xce\xd2\xce\xb1\xce\xd3.\xb0.\xd2~g\xfdA\xbb\xc4\xfa\x93v\x99u\x85v\x95u\x8dv\x9du\x83\xf6\x17\xebo\xdaM\xd6-\xda?\xac\x7fi\xb7YwhwY\xf7h\xf7Y\x0fh\x0fY\x8fh\x8fYOhOY\xcfh\xcfY/h/Y\xafh\xff\xb1^\xd3\xde\xb0\xde\xd2\xde\xc9\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0b\xc0\x05\x82z|\xeen\x0bX \xec\x7f\xc6\xe4\x02%"
- ), (5, 16): (
- b"x\x01\x1d\xc9\x89\r\x10\x04\x0c\x86\xd1\x8e\xc2\x16 \x97\x059\xd6\xe8&t\x94n\xa1\\RQ`\x8dn\x02/6yI\xf3\x7f\x11\xff\xdf\xc3\x88f\x89G\x11\x0fH\x8afX\x8e\xf8M')\x9aa9\xe2\xb1NR4\xc3r\xc4\x13\x9d\xa4h\x86\xe5\x88\xa7:I\xd1\x0c\xcb\x11\xcft\x92\xa2\x19\x96#\x9e\xeb$E3,G\xfc\xae\x93\x14\xcd\xb0\x1c\x91:I\xd1\x0c\xcb\x11/t\x92\xa2\x19\x96#^\xea$E3,G\xfc\xa1\x93\x14\xcd\xb0\x1c\xf1J')\x9aa9\xe2\xb5NR4\xc3r\xc4\x1b\x9d\xa4h\x86\xe5\x88\xb7:I\xbf\xb3\x13\x7f\xfaY\xe2/\x9d\xa4h\x86\xe5\x88\xf7:I\xd1\x0c\xcb\x11\x1ft\x92\xa2\x19\x96#>\xea$E3,G|\xd2I\x8afX\x8e\xf8\xac\x93\x14\xcd\xb0\x1c\xf1\xb7NR4\xc3r\xc4\x17\x9d\xa4h\x86\xe5\x88\xd5I\x8afX\x8e\xf8G')\x9aa9\xe2\xabNR4\xc3r\xc4\xbf:I\xd1\x0c\xcb\x11\xff\xe9$E3,G|\xd3I\x8afX\x8e\xf8\xae\x93\x14\xcd\xb0\x1c\xf1C'\xe9\x9f\xbf\x00Gi\xed\x02"
- ,
- b'x\x01\xed\xddU\xd6\x96\x05\x00\x85\xd1\x9f\x16\xa4\x14\x04\xe9P\xa4K\x1a\xe9\x14\x10\t\xe9\x0eQ\xba\xc1\xa0\xbb\xbb\xa5\x1b)%\xa5;%\xa5\xbb;%\x94\x8eAp\xf1.\xd6\xfe\xf6\x0c\xce3\x81\x13\x16\xf6\x8e\xc2\x05+|\xd0"\x04+b\xd0"\x05+r\xd0\xa2\x04\xeb\x83\xf7]T[4\xdd\x87\xb6\xe8\xba\x18\xb6\x98\xbaX\xb6\xd8\xba\x8fl\x1f\xeb\xe2\xd8\xe2\xea>\xb1\xc5\xd3\xc5\xb7}\xaaK`K\xa8KdK\xacKbK\xaaKfK\xaeKaK\xa9\xfb\xcc\xf6\xb9.\x95\xed\x0b]j[\x1a]Z[:]z[\x06]F[&]f[\x16]V\xdb\x97\xbal\xb6\xec\xba\x1c\xb6\x9c\xba\\\xb6\xdc\xba<\xb6\xbc\xba\xafl\xf9t\xf9m\x05t\x05m\x85t\x85mEtEm\xc5t\xc5m%t%m_\xebJ\xd9J\xeb\xca\xd8\xbe\xd1\x95\xb5}\xab+g+\xaf\xab`\xab\xa8\xfb\xceVIW\xd9VEW\xd5VMW\xddVCW\xd3VKW\xdbVGW\xd7VOW\xdf\xd6@\xd7\xd0\xf6\xbd\xae\x91\xed\x07\xdd\x8f\xb6\xc6\xba&\xb6\xa6\xbaf\xb6\xe6\xba\x16\xb6\x96\xbaV\xb6\xd6\xba6\xb6\xb6\xbav\xb6\xf6\xba\x0e\xb6\x8e\xba\x9fl?\xeb~\xb1\xfd\xaa\xebd\xeb\xac\xebb\xeb\xaa\xeb\x16\x12h\x81\xee!\xa1\x02\xa1\x02j\x81\xb0w\xd5#X=\x83\xd6+X\xbd\x83\xd6\'X}\x83\xd6/X\xfd\xdfw\x03l\x03u\x83l\x83uClCu\xc3l\xc3u#l#u\xa3l\xa3uclcu\xe3l\xe3u\x13l\xbf\xe9&\xda&\xe9&\xdb\xa6\xe8\xa6\xda\xa6\xe9\xa6\xdbf\xe8f\xdaf\xe9f\xdb\xe6\xe8\xe6\xda\xe6\xe9~\xb7\xcd\xd7-\xb0-\xd4-\xb2-\xd6\xfda\xfbS\xb7\xc4\xb6T\xb7\xcc\xb6\\\xb7\xc2\xb6R\xf7\x97m\x95n\xb5m\x8dn\xadm\x9dn\xbdm\x83n\xa3m\x93n\xb3m\x8bn\xabm\x9bn\xbbm\x87n\xa7m\x97n\xb7\xedo\xdd\x1e\xdb^\xdd>\xdb~\xdd\x01\xdbA\xdd?\xb6C\xba\xc3\xb6#\xba\xa3\xb6c\xba\xe3\xb6\x13\xba\x93\xb6S\xba\xd3\xb63\xba\xb3\xb6s\xba\xf3\xb6\x0b\xba\x8b\xb6K\xba\xcb\xb6+\xba\xab\xb6k\xba\xeb\xb6\x1b\xba\x9b\xb6[\xba\xdb\xb6;\xba\xbb\xb6{\xba\xfb\xb6\x7fu\x0fl\x0fu\x8fl\x8fu\xff\xd9\xfe\xd7=\xb1=\xd5=\xb3=\xd7\xbd\xb0\xbd\xd4\xbd\xb2\xbd\xd6\xbd\t\t\xb4\x80z|\x1e\xda\x1d*\x10*\xd0\xfd-\x8c\x93\xc6\x0e'
- )}
+#
+# This file is generated by generate_luts.py. DO NOT EDIT.
+#
+
+mxfp_luts_compressed = \
+{(2, 1, 1, 'saturate'): (b'x\x01\x1d\xc9\xc1\r\xc00\x00\xc2@o\xd6\x8c\xc6f\xf5h\xb1\x828\xf1\x00^>X\x0c\xa7f1,\x7f'
+ b'\x13\x83\xfdY\xf4\x027\xf1\x0c\xfb',
+ b"x\x01\xed\xdd\t\r\xc00\x10\x03\xc1\xf4\xff[\xfehS\x1a'y\x96\x81\x87\x80[\x13\x01\x02"
+ b'\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80'
+ b'\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10'
+ b' @\x80\x00\x01\x02e\x05\x06\x11 @\x80@\xa2\xc0(\x02\xc9\x02\x93\x92\x05fe\x0b,J\x16X\x95-'
+ b'\xb0\x89\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04'
+ b'\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00'
+ b'\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x14\x14\xe8"@ U`\x17\x01\x02\x04\x08\x10 '
+ b'@\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @'
+ b'\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02u\x05\x0e\x11 @'
+ b'\x80@\xa2\xc0)\x02\xc9\x02\x97\x92\x05ne\x0b<J\x16x\x95-\xf0\x89\x00\x01\x02\x04\x08\x10'
+ b' @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 '
+ b'@\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04'
+ b'\x08\x14\x14H=>\xb7\x9b\x00\x81\xfe\x03\\M8\xf2'),
+ (2, 3, 1, 'saturate'): (b'x\x01\x1d\xcdQ\r\xc30\x10\x03\xd0\x830\x08\x85P\x04[ \x14B \x0cB \x04B \x14\xc1v\x10'
+ b'\x02!\x10\x02aO\xb3\xf4\xe4?;\xe2\x9fgD#\x89W\xc4A\xa1\xd2\xe8\x0cn\x92\xc9b\x13%\xe2\xc1'
+ b'\xc1I\xe1\xa2\xf2\xa6\xd1\x19\xdc$\x93\xc5&\x1a\x1fE\x12_[\x14*\x8d\xce\xe0&\x99,6\x91\xfe98'
+ b')\\T\xde4:\x83\x9bd\xb2\xd8\xf9\x03~S=\xdd',
+ b'x\x01\xed\xdd\x85qB\x01\x14DQ\x12\x88\x11#\xeeF\x84\xb8\xbb\xf6_\x15\xa9\x82\xd9\xf9\xf3'
+ b'\xce\xed`O\x03\xdbj\x89\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01'
+ b'\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @'
+ b'\x80\x00\x81\xc6\x08\x8c\x89\x00\x01\x02\x04*\n\x8c\x8b@e\x81\xb6*\x0bt\x94\x15\x98\xa8\xddd'
+ b'\xf5\xa6j7\xdd\xf4f\xb2u\xd3\xcdf\x9bK7\x9fm!\xddb\xb6^\xba\xa5l\xcb\xe9V\xb2\xad\xa6['
+ b'\xcb\xb6\x9en#\xdbf\xba\xadl\xdb\xe9v\xb2\xed\xa6\xdb\xcb\xb6\x9f\xee@\x04\x08\x10 @\x80'
+ b'\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10'
+ b' @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02'
+ b'\x04\x08\x10\x18\x91\xc0P\x04\x08T\x158\x14\x01\x02\x04\x08\x10 @\x80\x00\x01\x02'
+ b'\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80'
+ b'\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\xcd\x118\x12\x01\x02\x04\x08T\x14\xe8\x8b@e\x81c'
+ b'U\x168QV\xe0\xb4vg\xd5\x1b\xd4\xee\xbc\xe9]d\xbbLw\x95\xed:\xddM\xb6\xdbtw\xd9\xee\xd3=d{L'
+ b'\xf7\x94\xed9\xddK\xb6\xd7to\xd9\xde\xd3}d\xfbL\xf7\x95\xed;\xddO\xb6\xdft\x7f"@\x80\x00\x01'
+ b'\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @'
+ b'\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08'
+ b'\x10 @\x80\xc0\x88\x04\xaa\x1e\x9f\xdbM\x80\xc0\xf0\x1f\x9f\x0cK\xfb'),
+ (3, 2, 3, 'saturate'): (b'x\x01\x15\xcbA\x15BQ\x08\x05@\xa2XD\xc5&/\nQ(\xa2\xf2\x9b\x10\xc5\x91\xc3\xc0\xe2B\xc4\xbf'
+ b'\xean<"\x92\xa2\x19\x96xF\xdcH\x0eE3,\x91r\x92C\xd1\x0cK\xbc\xe4$\x87\xa2\x19\x96(\xfd'
+ b'\xb6?n(\x9aa\x89\xaf\x7f\x92C\xd1\x0cK\x8c\x9c\xe4P4\xc3\x12\x97\x9c\xe4P4\xc3^?\xc7\x8a;c',
+ b'x\x01\xed\xdd\xd7U\x15\x00\x00\x04Q$\x89HV@r\x90 Q\xb2\x92\xfb\xaf\xea\xd1\x02\x7f'
+ b'\x1c\xce\xdc\xe9`o\x03;4$\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01'
+ b'\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\xe0\xd3\t|'
+ b'\x11\x01\x02\x04\x08\x14\x05\x86E\xa0,0\xa2\xb2\xc0\xa8>V`\xac\xddx\xbd\xaf\xed&\xea}k7Y\xef'
+ b'{\xbb\xa9z\xd3\xedf\xea\xcd\xb6\x9b\xab7\xdfn\xa1\xde\x8fv?\xeb-\xb6[\xaa\xb7\xdc\xee'
+ b'W\xbd\x95v\xab\xf5\xd6\xda\xad\xd7\xdbh\xb7YoK\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10'
+ b' @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 '
+ b'@\x80\x00\x01\x02\x04\x08\x10 \xf0.\x81\x81\x08\x10\xa8\nl\x8b\x00\x01\x02\x04\x08\x10 @\x80'
+ b'\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10'
+ b' @\x80\x00\x01\x02\x04\x08\x10\xf8|\x02;"@\x80\x00\x81\xa2\xc0\xae\x08\x94\x05~\xab,\xb0'
+ b'\xa7\x8f\x15\xd8owP\xef\xb0\xdd\x9fzG\xed\x8e\xeb\x9d\xb4;\xadw\xd6\xee\xbc\xde\xdfv\x17'
+ b'\xf5.\xdb]\xd5\xbbnwS\xef\xb6\xdd]\xbd\x7f\xed\xfe\xd7\xbbo\xf7P\xef\xb1\xddS\xbd\xe7v/\xf5^'
+ b'E\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @'
+ b'\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\x04\x08\x10 @\x80\x00\x01\x02\xef\x12'
+ b'\xa8\x1e\x9f\xdbM\x80\xc0\xe0\r\xd9\xf2;{'),
+ (4, 3, 7, 'saturate'): (b'x\x01\x1d\xcd[\xb5\x90!\x10\x80Q"\x18\x81\x08\xbc{C-@\x04"\xd0\xe0\x10\x81\x08D\xe0\xdd\x1bj'
+ b'\x01"\x10\xe1o\xa0\xfb\xc8Z{\xcd\xbc|C\x08\xff\xdf\xdb\x10:\x9b\xf0.\x84H\xa6\xd2\x19'
+ b'L\x16\x9b\xc3\xe5!\xbc\x0f\xe1\r\x91D\xa6Pit\x06\x93\xc5\xe6py\x08\x1f\xf4D\x12\x99'
+ b'B\xa5\xd1\x19L\x16\x9b\xc3\xe5!|\xd4\x13Id\n\x95Fg0Yl\x0e\x97\x87\x90\xf5D\x12\x99B\xa5'
+ b'\xd1\x19L\x16\x9b\xc3\xe5!|\xd2\x13Id\n\x95Fg0Yl\x0e\x97\x87\xf0YO$\x91)T\x1a\x9d\xc1d\xb19'
+ b'\\\x1e\xc2\x17=\x91D\xa6Pit\x06\x93\xc5\xe6p_\xf7\x17}\xe7\xab\xc1&|s\x8bL\xa53\x98,'
+ b'6\x87\xcbC\xf8\xee\x7f"\x89L\xa1\xd2\xe8\x0c&\x8b\xcd\xe1\xf2\x10~\xe8\x89$2\x85J\xa33\x98,6'
+ b"\x87\xcbC\xf8\xa9'\x92\xc8\x14*\x8d\xce`\xb2\xd8\x1c.\x0fa\xeb\x89$2\x85J\xa33\x98,6\x87\xcb"
+ b'C\xf8\xa5\'\x92\xc8\x14*\x8d\xce`\xb2\xd8\x1c.\x0f\xe1\xb7\x9eH"S\xa84:\x83\xc9bs\xb8<\x84'
+ b'?z"\x89L\xa1\xd2\xe8\x0c&\x8b\xcd\xe1\xbe\xee\x7f\xff\x01!\xba\xf7\x9b',
+ b'x\x01\xed\xdd\xd5\x96\x90\x05\x00E\xe1\x01\xe9\x90n\x90\x96F\xbaE\xa4S@\xba\x15\xa4\xbbKX H*'
+ b'\xdd)\xa9\x12Jw*H\x08Jww\x87\xd25\\\xccC\xec5\xf3\xef\xef\r\xce~\x81\x13\x12"\x0bX'
+ b'\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX \xc2\x16\x88$\x0b'
+ b'X\xc0\x02\x16\x08b\x81\xc8\xb2@\x90\x0b|\xa0 \x17\x88"\xb6@\xd4`\x8b\x16t\xd1\x83-Fx\x17\x93'
+ b'\x15\x8b\x16\x9b\x15\x87\x16\x97\xf5!-\x1e+>-\x01+!-\x11+1-\t+)-\x19+9-\x05+%-\x15+5-\r'
+ b'\xeb#ZZV:ZzV\x06ZFV&Zf\xd6\xc7\xb4,\xac\xac\xb4l\xac\xec\xb4\x1c\xac\x9c\xb4\\\xac\xdc\xb4O'
+ b'XyhyY\xf9h\xf9Y\x05h\x05Y\x85h\x85YEhEY\xc5h\xc5Y%h\x9f\xb2J\xd2>c\x95\xa2}\xce*M+\xc3*K+'
+ b'\xc7*O\xab\xc0\xaaH\xab\xc4\xaaL\xab\xc2\xaaJ\xab\xc6\xfa\x82V\x9dU\x83V\x93\xf5%\xad'
+ b'\x16\xab6\xad\x0e\xab.\xad\x1e\xab>\xad\x01\xab!\xad\x11\xab1\xad\t\xab)\xad\x19\xeb+\xda'
+ b'\xd7\xac\xe6\xb4\x16\xacoh-Y\xadh\xadYmhmY\xedh\xedY\x1dh\x1dY\x9dh\x9dY]h]Y\xddh\xddY=h'
+ b'=Y\xbdh\xbdY}h}Y\xfdh\xfdY\xdf\xd2\x06\xb0\x06\xca\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02'
+ b'\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05\xc2\n'
+ b'\x84\xca\x02\x16\x08j\x81A\xb2\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0'
+ b'\x02\x16\xb0\x80\x05"n\x81\xc1\xb2\x80\x05,`\x81 \x16\xf8N\x16\x08r\x81!\nr\x81\xa1'
+ b'b\x0b|\x1fl\xc3\x82nx\xb0\x8d\x08\xefF\xb2F\xd1F\xb3~\xa0\xfd\xc8\x1aC\x1b\xcb\x1a'
+ b'G\x1b\xcf\x9a@\x9b\xc8\x9aD\x9b\xcc\x9aB\x9b\xca\x9aF\x9b\xce\x9aA\x9b\xc9\x9aE\x9b\xcd\x9a'
+ b'C\x9b\xcb\xfa\x896\x8f5\x9f\xb6\x80\xb5\x90\xb6\x88\xb5\x98\xf63\xeb\x17\xda\xaf\xac'
+ b'%\xb4\xa5\xace\xb4\xe5\xac\xdfh\xbf\xb3V\xd0V\xb2V\xd1V\xb3\xd6\xd0\xd6\xb2\xd6\xd1\xd6\xb3'
+ b'6\xd06\xb26\xd16\xb3\xb6\xd0\xb6\xb2\xb6\xd1\xb6\xb3v\xd0v\xb2\xfe\xa0\xfd\xc9\xdaE\xdb\xcd'
+ b'\xfa\x8b\xb6\x87\xb5\x97\xb6\x8f\xb5\x9f\xf67\xeb\x00\xed \xeb\x1f\xda\xbf\xacC\xb4\xc3'
+ b'\xac#\xb4\xa3\xacc\xb4\xe3\xac\x13\xb4\x93\xacS\xb4\xd3\xac3\xb4\xb3\xacs\xb4\xf3'
+ b'\xac\x0b\xb4\x8b\xacK\xb4\xcb\xac+\xb4\xab\xack\xb4\xeb\xac\x1b\xb4\x9b\xac[\xb4\xdb'
+ b'\xac;\xb4\xbb\xac{\xb4\xfb\xac\x07\xb4\x87\xacG\xb4\xc7\xac\xffh\xff\xb3\x9e\xd0\x9e'
+ b'\xb2\x9e\xd1\x9e\xb3^\xd0^\xb2^\xd1^\xb3\xde\xd0\xde\xb2\xde\xc9\x02\x16\xb0\x80\x05'
+ b',`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0'
+ b'\x02\x16\xb0\x80\x05\xc2\n\x04\xf5\xf8\xdc\xdd\x16\xb0@\xe8{\t?\xc8\x90'),
+ (5, 2, 15, 'saturate'): (b'x\x01M\xcb\xd9\x11\x10T\x0c@\xd1\x94B\x17 \x08\x06Dh#\x9d\x90R\xd2\x05\xca\x1a\x17'
+ b'\xa0\x8dt\x02\xc7\x1f\xc7\xcc\x9c\xf7\xf1n\x12\xf1\xef\xf4C\xcf\xa3\x88\xa4\x19\x96#~\x8ax@'
+ b"R4\xc3r\xc4c\x9d\xa4h\x86\xe5\x88':I\xd1\x0c\xcb\x11?\xeb$E3,G<\xd5I\x8afX\x8ex\xa6\x93"
+ b'\x14\xcd\xb0\x1c\xf1\x8bNR4\xc3rD\xea$E3,G<\xd7I\x8afX\x8ex\xa1\x93\x14\xcd\xb0\x1c'
+ b'\xf1\xabNR4\xc3r\xc4K\x9d\xa4h\x86\xe5\x88\xdft\x92\xa2\x19\x96#^\xe9$E3,G\xbc\xd6I'
+ b'\x8a~\xa3\xfdO\xb4\xbf\xb7\xf6~\xb7C3,G\xfc\xe1\x9e\xa4h\x86\xe5\x88w:I\xd1\x0c'
+ b'\xcb\x11\xefu\x92\xa2\x19\x96#>\xe8$E3,G|\xd4I\x8afX\x8e\xf8\xa4\x93\x14\xcd\xb0\x1c\xf1Y'
+ b"')\x9aa9bu\x92\xa2\x19\x96#\xfe\xd4I\x8afX\x8e\xf8K')\x9aa9\xe2o\x9d\xa4h\x86\xe5\x88\x7ft"
+ b'\x92\xa2\x19\x96#\xbe\xe8$E3,G|\xd5I\x8afX\x8e\xf8\xa6\x93\x14\xfd]\xfb\xcf\x0f'
+ b'\xd2\x15\xf0\xcf',
+ b'x\x01\xed\xddC\xa2\x1c\x00\x00D\xc1\xd8\xb6m\xdb\xb6m\xdb66\xb1m\xdb\xb6m\xdb\xb6m'
+ b',\x92c\xfcEM\xdd\xa0\xdf\x05:X\xb0 \x16\xdc\x16B\x17\xd2\x16J\x17\xda\x16F\x17\xd6'
+ b'\x16N\x17\xde\x16A\x17\xd1\x16I\x17\xd9\x16E\x17\xd5\x16M\x17\xdd\x16C\x17\xd3\x16K\x17\xdb'
+ b'\x16G\x17\xd7\x16O\x17\xdf\x96@\x97\xd0\x96H\x97\xd8\x96D\x97\xd4\x96L\x97\xdc\x96B\x97\xd2'
+ b'\x96J\x97\xda\x96F\x97\xd6\x96N\x97\xde\x96A\x97\xd1\x96I\x97\xd9\x96E\x97\xd5\x96M\x97\xdd'
+ b'\x96C\x97\xd3\x96K\x97\xdb\x96G\x97\xd7\x96O\x97\xdfV@W\xd0VHW\xd8VDW\xd4VLW\xdcVBW\xd2'
+ b'VJW\xdaVFW\xd6VNW\xdeVAW\xd1VIW\xd9VEW\xd5VMW\xddVCW\xd3VKW\xdbVGW\xd7VOW\xdf\xd6@\xd7\xd0'
+ b'\xd6H\xd7\xd8\xd6D\xd7\xd4\xd6L\xd7\xdc\xd6B\xd7\xd2\xd6J\xd7\xda\xd6F\xd7\xd6\xd6N\xd7\xde'
+ b'\xd6A\xd7\xd1\xd6I\xd7\xd9\xd6E\xd7\xd5\xd6M\xd7\xdd\xd6C\xd7\xd3\xd6K\xd7\xdb\xd6G\xd7\xd7'
+ b'\xd6O\xd7\xdf6@70 H\x0b\xfc\x0b\x08\x14\x08\x14P\x0b\x0c\nj\x83mCtCm\xc3t\xc3m#t#m'
+ b'\xa3t\xa3mctcm\xe3t\xe3m\x13t\x13m\x93t\x93mStSm\xd3t\xd3m3t3m\xb3t\xb3mstsm\xf3t\xf3m'
+ b'\x0bt\x0bm\x8bt\x8bmKtKm\xcbt\xcbm+t+m\xabt\xabmktkm\xebt\xebm\x1bt\x1bm\x9bt\x9bm[t[m'
+ b"\xdbt\xdbm;t;m\xbbt\xbbm{t{m\xfbt\xfbm\x07t\x07m\x87t\x87mGtGm\xc7t\xc7m't'm\xa7t\xa7mgtgm"
+ b'\xe7t\xe7m\x17t\x17m\x97t\x97mWtWm\xd7t\xd7m7t7m\xb7t\xb7mwtwm\xf7t\xf7m\x0ft\x0fm'
+ b'\x8ft\x8fmOtOm\xcft\xcfm/t/m\xaft\xafmotom\xeft\xefm\x1ft\x1fm\x9ft\x9fm_t_m\xdft\xdfm?t?m'
+ b'\xbft\xbfm\x7ft\x7f\x03\x82\xb4\x80z|\x1e\xd8\x1d(\x10(\xf0\xef?\xe6\xfc\r\x9b'),
+ (4, 3, 7, 'overflow'): (b'x\x01\x1d\xcd[\xb5\x90!\x10\x80Q"\x18\x81\x08\xbc{C-@\x04"\xd0\xe0\x10\x81\x08D\xe0\xdd\x1bj'
+ b'\x01"\x10\xe1o\xa0\xfb\xc8Z{\xcd\xbc|C\x08\xff\xdf\xdb\x10:\x9b\xf0.\x84H\xa6\xd2\x19'
+ b'L\x16\x9b\xc3\xe5!\xbc\x0f\xe1\r\x91D\xa6Pit\x06\x93\xc5\xe6py\x08\x1f\xf4D\x12\x99'
+ b'B\xa5\xd1\x19L\x16\x9b\xc3\xe5!|\xd4\x13Id\n\x95Fg0Yl\x0e\x97\x87\x90\xf5D\x12\x99B\xa5'
+ b'\xd1\x19L\x16\x9b\xc3\xe5!|\xd2\x13Id\n\x95Fg0Yl\x0e\x97\x87\xf0YO$\x91)T\x1a\x9d\xc1d\xb19'
+ b'\\\x1e\xc2\x17=\x91D\xa6Pit\x06\x93\xc5\xe6p_\xf7\x17}\xe7\xab\xc1&|s\x8bL\xa53\x98,'
+ b'6\x87\xcbC\xf8\xee\x7f"\x89L\xa1\xd2\xe8\x0c&\x8b\xcd\xe1\xf2\x10~\xe8\x89$2\x85J\xa33\x98,6'
+ b"\x87\xcbC\xf8\xa9'\x92\xc8\x14*\x8d\xce`\xb2\xd8\x1c.\x0fa\xeb\x89$2\x85J\xa33\x98,6\x87\xcb"
+ b'C\xf8\xa5\'\x92\xc8\x14*\x8d\xce`\xb2\xd8\x1c.\x0f\xe1\xb7\x9eH"S\xa84:\x83\xc9bs\xb8<\x84'
+ b'?z"\x89L\xa1\xd2\xe8\x0c&\x8b\xcd\xe1\xbe\xee\x7f\xff\x01!\xba\xf7\x9b',
+ b'x\x01\xed\xdc\x05\xb2\x90\x05\x00E\xe1\x87\x80\xa4t\x83tw\x97Jw\x0bHKw\x83\x80\x92Cww7'
+ b'\x92\xd2\x1dRJww\x97\xa4\x92J\xba\x8c3\x8f\xff|\x1b\xb83g\x017$D\x16\xb0\x80\x05,`\x01'
+ b'\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\xc0g[ \x8c,`\x01\x0bX '
+ b'\x88\x05\xbe\x90\x05\x82\\ \xac\x82\\ \x9c\xd8\x02\xe1\x83\xed\xcb\xa0\x8b\x10l\x11'
+ b'C\xbbH\xac\xc8\xb4(\xac\xa8\xb4\xafX\xd1h\xd1Y1h1Y\xb1h\xb1YqhqY\xf1h\xf1Y\th\tY\x89h\x89Y'
+ b'Ih_\xb3\x92\xd2\x92\xb1\x92\xd3R\xb0R\xd2R\xb1R\xd3\xd2\xb0\xd2\xd2\xd2\xb1\xd2\xd32\xb0'
+ b'2\xd22\xb12\xd3\xb2\xb0\xb2\xd2\xb2\xb1\xb2\xd3r\xb0r\xd2r\xb1r\xd3\xf2\xb0\xf2\xd2\xf2\xb1'
+ b'\xf2\xd3\n\xb0\n\xd2\xbea}K\xfb\x8eU\x88V\x98U\x84V\x94U\x8cV\x9cU\x82V\x92U\x8aV\x9a'
+ b'U\x86V\x96U\x8eV\x9eU\x81V\x91U\x89V\x99U\x85\xf6=\xab*\xad\x1a\xab:\xed\x07V\rZMV-ZmV\x1dZ]'
+ b'V=\xda\x8f\xac\xfa\xb4\x06\xac\x86\xb4F\xac\xc6\xb4&\xac\xa6\xb4f\xac\xe6\xb4\x16'
+ b'\xac\x96\xb4V\xac\xd6\xb46\xac\xb6\xb4v\xac\xf6\xb4\x0e\xac\x8e\xb4N\xac\xce\xb4\x9fX]h]'
+ b'Y\xddh?\xb3~\xa1ug\xf5\xa0\xf5d\xf5\xa2\xf5f\xf5\xa1}\x92\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80'
+ b'\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX'
+ b'\xc0\x02\x16`\x0b\xf4\x95\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0'
+ b"\x80\x05,\xf0\xf9\x16\xe8'\x0bX\xc0\x02\x16\x08b\x81\xfe\xb2@\x90\x0b\x0cP\x90\x0b\x0c\x14["
+ b'`P\xb0\r\x0e\xba!\xc164\xb4\x1b\xc6\x1aN\x1b\xc1\x1aI\x1b\xc5\x1aM\x1b\xc3\x1aK\x1b'
+ b'\xc7\x1aO\x9b\xc0\x9aH\x9b\xc4\x9aL\x9b\xc2\x9aJ\x9b\xc6\x9aN\x9b\xc1\x9aI\x9b\xc5\x9aM\x9b'
+ b'\xc3\x9aK\x9b\xc7\x9aO[\xc0ZH[\xc4\xfa\x95\xb6\x98\xb5\x84\xb6\x94\xb5\x8c\xb6'
+ b'\x9c\xb5\x82\xf6\x1bk%m\x15k5m\rk-m\x1dk=m\x03k#m\x13k3m\x0bk+m\x1bk;m\x07\xebw\xda'
+ b"N\xd6.\xdan\xd6\x1e\xda^\xd6\x1f\xb4?Y\xfbh\xfbY\x07h\x07Y\x87h\x87YGhGY\xc7h\xc7Y'h'Y\xa7h"
+ b'\xa7YghgY\xe7h\xe7Y\x17h\x17Y\x97h\x97YWhWY\xd7h\xd7Y7h7Y\xb7h\xb7YwhwY\xf7h\xf7Y\x0fh'
+ b'\x7f\xb1\x1e\xd2\x1e\xb1\x1e\xd3\x9e\xb0\x9e\xd2\x9e\xb1\xfe\xa6\xfd\xc3zN{\xc1zI{\xc5zM'
+ b'{\xc3\xfa\x97\xf6\x1f\xeb-\xed\x1d\xeb=\xed\x03\xeb#\x8d\xbd\xefw\xdd\x02\x16\xb0\x80\x05,`'
+ b'\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16'
+ b'\xb0\x80\x05,`\x01\x0b|\xfa\x1f\xb2\xf6b\xf1'),
+ (5, 2, 15, 'overflow'): (b'x\x01M\xcb\xd9\x11\x10T\x0c@\xd1\x94B\x17 \x08\x06Dh#\x9d\x90R\xd2\x05\xca\x1a\x17'
+ b'\xa0\x8dt\x02\xc7\x1f\xc7\xcc\x9c\xf7\xf1n\x12\xf1\xef\xf4C\xcf\xa3\x88\xa4\x19\x96#~\x8ax@'
+ b"R4\xc3r\xc4c\x9d\xa4h\x86\xe5\x88':I\xd1\x0c\xcb\x11?\xeb$E3,G<\xd5I\x8afX\x8ex\xa6\x93"
+ b'\x14\xcd\xb0\x1c\xf1\x8bNR4\xc3rD\xea$E3,G<\xd7I\x8afX\x8ex\xa1\x93\x14\xcd\xb0\x1c'
+ b'\xf1\xabNR4\xc3r\xc4K\x9d\xa4h\x86\xe5\x88\xdft\x92\xa2\x19\x96#^\xe9$E3,G\xbc\xd6I'
+ b'\x8a~\xa3\xfdO\xb4\xbf\xb7\xf6~\xb7C3,G\xfc\xe1\x9e\xa4h\x86\xe5\x88w:I\xd1\x0c'
+ b'\xcb\x11\xefu\x92\xa2\x19\x96#>\xe8$E3,G|\xd4I\x8afX\x8e\xf8\xa4\x93\x14\xcd\xb0\x1c\xf1Y'
+ b"')\x9aa9bu\x92\xa2\x19\x96#\xfe\xd4I\x8afX\x8e\xf8K')\x9aa9\xe2o\x9d\xa4h\x86\xe5\x88\x7ft"
+ b'\x92\xa2\x19\x96#\xbe\xe8$E3,G|\xd5I\x8afX\x8e\xf8\xa6\x93\x14\xfd]\xfb\xcf\x0f'
+ b'\xd2\x15\xf0\xcf',
+ b'x\x01\xed\xddS\xb6\x1e\x06\x00\x85\xd1\x1b\xdb\xb6\x1b\xdb\xb6\x9b\x06\x8dm\xdbz\x88'
+ b"m\xabI\x1b\xa3\xb1m\xdb\xb6mg\xad\x9ba\xdc\x87\xfd\xef\x19\x9co\x02'((\x84\x85\xb2"
+ b'\x85\xd6\x85\xb1\x85\xd5\x85\xb3\x85\xd7E\xb0E\xd4E\xb2E\xd6E\xb1E\xd5E\xb3E\xd7\xc5\xb0'
+ b'\xc5\xd4\xc5\xb2\xc5\xd6\xc5\xb1\xc5\xd5\xc5\xb3\xc5\xd7%\xb0%\xd4%\xb2%\xd6%\xb1%\xd5%\xb3'
+ b'%\xd7\xa5\xb0\xa5\xd4\xa5\xb2\xa5\xd6\xa5\xb1\xa5\xd5\xa5\xb3\xfd\xa1Ko\xcb\xa0\xcbh'
+ b'\xcb\xa4\xcbl\xcb\xa2\xcbj\xcb\xa6\xcbn\xcb\xa1\xcbi\xcb\xa5\xcbm\xcb\xa3\xcbk\xcb\xa7\xcbo'
+ b'+\xa0+h+\xa4+l+\xa2+j+\xa6+n+\xa1+i+\xa5+m+\xa3+k+\xa7+o\xab\xa0\xabh\xab\xa4\xabl'
+ b'\xfbSW\xc5\xf6\x97\xae\xaa\xad\x9a\xae\xba\xad\x86\xeeo[M]-[m]\x1d[]]=[}]\x03[C]#[c]\x13'
+ b"[S]3[s]\x0b[K]+[k]\x1b[[];[{]\x07[G]'[g]\x17[W]7[w]\x0f[O]/[o]\x1f[_]?[\x7f\xdd\x00"
+ b'\xdb\xc0\x90\x16\x1c\x10(\x10(\xa0\x16\x18\x14\xd2\x06\xdb\x86\xe8\x86\xda\x86\xe9\x86\xdb'
+ b'F\xe8F\xdaF\xe9F\xdb\xc6\xe8\xc6\xda\xc6\xe9\xc6\xdb&\xe8&\xda&\xe9&\xdb\xa6\xe8\xa6\xda'
+ b'\xa6\xe9\xa6\xdbf\xe8f\xdaf\xe9f\xdb\xe6\xe8\xe6\xda\xe6\xe9\xfe\xb1\xcd\xd7-\xb0'
+ b'\xfd\xab\xfb\xcf\xb6P\xb7\xc8\xb6X\xb7\xc4\xb6T\xb7\xcc\xb6\\\xb7\xc2\xb6R\xb7\xca'
+ b'\xf6\xbfn\xb5m\x8dn\xadm\x9dn\xbdm\x83n\xa3m\x93n\xb3m\x8bn\xabm\x9bn\xbbm\x87n\xa7'
+ b'm\x97n\xb7m\x8fn\xafm\x9fn\xbf\xed\x80\xee\xa0\xed\x90\xee\xb0\xed\x88\xee\xa8'
+ b'\xed\x98\xee\xb8\xed\x84\xee\xa4\xed\x94\xee\xb4\xed\x8c\xee\xac\xed\x9c\xee\xbc'
+ b'\xed\x82\xee\xa2\xed\x92\xee\xb2\xed\x8a\xee\xaa\xed\x9a\xee\xba\xed\x86\xee\xa6'
+ b'\xed\x96\xee\xb6\xed\x8e\xee\xae\xed\x9e\xee\xbe\xed\x81\xee\xa1\xed\x91\xee\xb1'
+ b'\xed\x89\xee\xa9\xed\x99\xee\xb9\xed\x85\xee\xa5\xed\x95\xee\xb5\xed\x8d\xee\xad'
+ b'\xed\x9d\xee\xbd\xed\x83\xee\xa3\xed\x93\xee\xb3\xed\x8b\xee\xab\xed\x9b\xee\xbb'
+ b'\xed\x87\xee\xa7\xedWHS\x8f\xcf\x03\xbb\x03\x05\x02\x05\x82\x7f\x03\xb3\x87\x0e\x9d')}
+
+
+binary8_luts_compressed = \
+{(4, 8): (b'x\x01\x15\xcb[\xd5P!\x10\x80Q"\x18\x81\x08<{E-@\x04"\xd0@"\x10\x81\x08<{E\xff\x02\'\x02\x11h\xa0\xdbY'
+ b'k\xcf\xcb\xcc\x17\xc2\xff\xe9\xaf\xad7!d:\x93\xcd!\xbc\r\xe1\x15\x91D\xa6Pit\x06\x93\xc5\xe6\xe1p\t\xef\xf4'
+ b'D\x12\x99B\xa5\xd1\x19L\x16\x9b\x87\xc3%\xbc\xd7\x13Id\n\x95Fg0Yl\x1e\x0e\x97\xf0AO$\x91)T\x1a\x9d\xc1d\xb1'
+ b"y8\\B\xd6\x13Id\n\x95Fg0Yl\x1e\x0e\x97\xf0QO$\x91)T\x1a\x9d\xc1d\xb1y8\\\xc2'=\x91D\xa6Pit\x06\x93"
+ b'\xc5\xe6\xe1p\t\x9f\xf5D\x12\x99B\xa5\x7f\xf1O\xff\xea\xef\x9b\x1b\x9d\xc9\xe6\x10\xbe\xeb\x89$2\x85J\xa3'
+ b'3\x98,6\x0f\x87K\xf8\xa1\'\x92\xc8\x14*\x8d\xce`\xb2\xd8<\x1c.\xe1\xa7\x9eH"S\xa84:\x83\xc9b\xf3p\xb8\x84_z'
+ b'"\x89L\xa1\xd2\xe8\x0c&\x8b\xcd\xc3\xe1\x12\xb6\x9eH"S\xa84:\x83\xc9b\xf3p\xb8\x84\xdfz"\x89L\xa1\xd2\xe8'
+ b'\x0c&\x8b\xcd\xc3\xe1\x12\xfe\xe8\x89$2\x85J\xa33\x98,6\x0f\x87Kx\xd1\x13Id\n\x95\xfe\xf7\x1f[)\xf3`',
+ b'x\x01\xed\xdd\x05\xba\x96\x05\x00\x05\xe1\x9f\xee\x06\xe9FZA\xa4\xbb\xbb;\xa4SB\xba\xeb\xd2\xdd\x8dt\x97'
+ b'\x92J(\xa14\xa2\x84\x92\x8a\xa4\x82\xd2\x1d\x12.c\x9e\xcb7\xef\x0e\xcel\xe0\x84B\xb2\x80\x05,`\x01'
+ b'\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xe0\xbd/\x10A\x16\xb0\x80\x05,\x10\xc4\x02\x11e\x81 \x17\x88'
+ b'\xa4 \x17\x88,\xb6@\x94`\x8b\x1at\xd1\x82-zx\x17\x83\x15\x93\x16\x8b\x15\x9b\x16\x87\x15\x97\x16\x8f\x15'
+ b'\x9f\x96\x80\x95\x90\x96\x88\x95\x98\x96\x84\xf5\x01-)+\x19-9+\x05-%+\x15-5+\r--+\x1d-=+\x03-#+\x13\xedCV'
+ b'fZ\x16VVZ6VvZ\x0eVN\xdaG\xac\x8fi\xb9X\xb9i\x9f\xb0\xf2\xd0>e\xe5\xa5\xe5c\xe5\xa7\x15`\x15\xa4\x15b'
+ b'\x15\xa6\x15a\x15\xa5\x15c\x15\xa7\x95`\x95\xa4\x95b\x95\xa6\x95a\x95\xa5\x95c\x95\xa7U`U\xa4UbU\xa6Ua'
+ b'U\xa5UcU\xa7\xd5`\xd5\xa4\xd5b\xd5\xa6\xd5a\xd5\xa5\xd5c\xd5\xa75`5\xa45b5\xa65a}Fk\xcajFk\xcejAk\xc9'
+ b"jEk\xcdjCk\xcbjGk\xcf\xea@\xfb\x9c\xd5\x91\xd6\x89\xd5\x99\xd6\x85\xf5\x05\xad+\xab\x1b\xad;\xab\x07\xad'"
+ b'\xab\x17\xad7\xab\x0f\xad/\xab\x1f\xad?k\x00m k\x10m0k\x08m\xa8,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,'
+ b'`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x00\\'
+ b' L\x16\xb0@P\x0b\xbc\xf7\xff\x86\x0e\xb4\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX '
+ b'\x14\x1a&\x0bX\xc0\x02\x16\x08b\x81\xe1\xb2@\x90\x0b\x8cP\x90\x0b\x8c\x14[`T\xb0\x8d\x0e\xba1\xc16'
+ b'6\xbc\x1b\xc7\x1aO\x9b\xc0\x9aH\x9b\xc4\x9aL\x9b\xc2\x9aJ\x9b\xc6\x9aN\x9b\xc1\x9aI\x9b\xc5\x9aM\x9b\xc3'
+ b'\x9aK\x9b\xc7\x9aO\xfb\x92\xb5\x80\xb6\x90\xb5\x88\xb6\x98\xb5\x84\xb6\x94\xb5\x8c\xb6\x9c\xb5\x82\xb6\x92'
+ b'\xb5\x8a\xb6\x9a\xb5\x86\xb6\x96\xb5\x8e\xb6\x9e\xb5\x81\xf6\x15\xebk\xdaF\xd6&\xdaf\xd6\x16\xdaV\xd66\xda7'
+ b'\xacoi\xdbY;h;Y\xbbh\xdf\xb1\xbe\xa7\xedf\xed\xa1\xede\xed\xa3\xfd\xc0\xfa\x91\xb6\x9fu\x80v\x90u\x88v'
+ b'\x98u\x84v\x94u\x8c\xf6\x13\xeb8\xedg\xd6/\xb4\x13\xac\x93\xb4S\xac\xd3\xb4_Y\xbf\xd1\xce\xb0\xce\xd2'
+ b'\xce\xb1\xce\xd3.\xb0.\xd2~g\xfdA\xbb\xc4\xfa\x93v\x99u\x85v\x95u\x8dv\x9du\x83\xf6\x17\xebo\xdaM\xd6-'
+ b'\xda?\xac\x7fi\xb7YwhwY\xf7h\xf7Y\x0fh\x0fY\x8fh\x8fYOhOY\xcfh\xcfY/h/Y\xafh\xff\xb1^\xd3\xde\xb0\xde'
+ b'\xd2\xde\xc9\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`'
+ b'\x01\x0bX\xc0\x02\x16\xb0\x80\x05,`\x01\x0b\xc0\x05\x82z|\xeen\x0bX \xec\x7f\xc6\xe4\x02%'),
+ (5, 16): (b'x\x01\x1d\xc9\x89\r\x10\x04\x0c\x86\xd1\x8e\xc2\x16 \x97\x059\xd6\xe8&t\x94n\xa1\\RQ`\x8dn\x02/6yI'
+ b"\xf3\x7f\x11\xff\xdf\xc3\x88f\x89G\x11\x0fH\x8afX\x8e\xf8M')\x9aa9\xe2\xb1NR4\xc3r\xc4\x13\x9d\xa4h"
+ b'\x86\xe5\x88\xa7:I\xd1\x0c\xcb\x11\xcft\x92\xa2\x19\x96#\x9e\xeb$E3,G\xfc\xae\x93\x14\xcd\xb0\x1c\x91'
+ b":I\xd1\x0c\xcb\x11/t\x92\xa2\x19\x96#^\xea$E3,G\xfc\xa1\x93\x14\xcd\xb0\x1c\xf1J')\x9aa9\xe2\xb5NR4\xc3"
+ b'r\xc4\x1b\x9d\xa4h\x86\xe5\x88\xb7:I\xbf\xb3\x13\x7f\xfaY\xe2/\x9d\xa4h\x86\xe5\x88\xf7:I\xd1\x0c\xcb'
+ b'\x11\x1ft\x92\xa2\x19\x96#>\xea$E3,G|\xd2I\x8afX\x8e\xf8\xac\x93\x14\xcd\xb0\x1c\xf1\xb7NR4\xc3r'
+ b"\xc4\x17\x9d\xa4h\x86\xe5\x88\xd5I\x8afX\x8e\xf8G')\x9aa9\xe2\xabNR4\xc3r\xc4\xbf:I\xd1\x0c\xcb\x11"
+ b"\xff\xe9$E3,G|\xd3I\x8afX\x8e\xf8\xae\x93\x14\xcd\xb0\x1c\xf1C'\xe9\x9f\xbf\x00Gi\xed\x02",
+ b'x\x01\xed\xddU\xd6\x96\x05\x00\x85\xd1\x9f\x16\xa4\x14\x04\xe9P\xa4K\x1a\xe9\x14\x10\t\xe9\x0eQ'
+ b'\xba\xc1\xa0\xbb\xbb\xa5\x1b)%\xa5;%\xa5\xbb;%\x94\x8eAp\xf1.\xd6\xfe\xf6\x0c\xce3\x81\x13\x16\xf6'
+ b'\x8e\xc2\x05+|\xd0"\x04+b\xd0"\x05+r\xd0\xa2\x04\xeb\x83\xf7]T[4\xdd\x87\xb6\xe8\xba\x18\xb6\x98\xbaX\xb6'
+ b'\xd8\xba\x8fl\x1f\xeb\xe2\xd8\xe2\xea>\xb1\xc5\xd3\xc5\xb7}\xaaK`K\xa8KdK\xacKbK\xaaKfK\xaeKaK\xa9\xfb\xcc'
+ b'\xf6\xb9.\x95\xed\x0b]j[\x1a]Z[:]z[\x06]F[&]f[\x16]V\xdb\x97\xbal\xb6\xec\xba\x1c\xb6\x9c\xba\\'
+ b'\xb6\xdc\xba<\xb6\xbc\xba\xafl\xf9t\xf9m\x05t\x05m\x85t\x85mEtEm\xc5t\xc5m%t%m_\xebJ\xd9J\xeb\xca'
+ b'\xd8\xbe\xd1\x95\xb5}\xab+g+\xaf\xab`\xab\xa8\xfb\xceVIW\xd9VEW\xd5VMW\xddVCW\xd3VKW\xdbVGW\xd7VOW'
+ b'\xdf\xd6@\xd7\xd0\xf6\xbd\xae\x91\xed\x07\xdd\x8f\xb6\xc6\xba&\xb6\xa6\xbaf\xb6\xe6\xba\x16\xb6\x96\xba'
+ b'V\xb6\xd6\xba6\xb6\xb6\xbav\xb6\xf6\xba\x0e\xb6\x8e\xba\x9fl?\xeb~\xb1\xfd\xaa\xebd\xeb\xac\xebb\xeb\xaa'
+ b"\xeb\x16\x12h\x81\xee!\xa1\x02\xa1\x02j\x81\xb0w\xd5#X=\x83\xd6+X\xbd\x83\xd6'X}\x83\xd6/X\xfd\xdfw"
+ b'\x03l\x03u\x83l\x83uClCu\xc3l\xc3u#l#u\xa3l\xa3uclcu\xe3l\xe3u\x13l\xbf\xe9&\xda&\xe9&\xdb\xa6\xe8'
+ b'\xa6\xda\xa6\xe9\xa6\xdbf\xe8f\xdaf\xe9f\xdb\xe6\xe8\xe6\xda\xe6\xe9~\xb7\xcd\xd7-\xb0-\xd4-\xb2-\xd6'
+ b'\xfda\xfbS\xb7\xc4\xb6T\xb7\xcc\xb6\\\xb7\xc2\xb6R\xf7\x97m\x95n\xb5m\x8dn\xadm\x9dn\xbdm\x83n\xa3m\x93'
+ b'n\xb3m\x8bn\xabm\x9bn\xbbm\x87n\xa7m\x97n\xb7\xedo\xdd\x1e\xdb^\xdd>\xdb~\xdd\x01\xdbA\xdd?\xb6C'
+ b'\xba\xc3\xb6#\xba\xa3\xb6c\xba\xe3\xb6\x13\xba\x93\xb6S\xba\xd3\xb63\xba\xb3\xb6s\xba\xf3\xb6\x0b'
+ b'\xba\x8b\xb6K\xba\xcb\xb6+\xba\xab\xb6k\xba\xeb\xb6\x1b\xba\x9b\xb6[\xba\xdb\xb6;\xba\xbb\xb6{'
+ b'\xba\xfb\xb6\x7fu\x0fl\x0fu\x8fl\x8fu\xff\xd9\xfe\xd7=\xb1=\xd5=\xb3=\xd7\xbd\xb0\xbd\xd4\xbd\xb2\xbd'
+ b'\xd6\xbd\t\t\xb4\x80z|\x1e\xda\x1d*\x10*\xd0\xfd-\x8c\x93\xc6\x0e')}
diff --git a/bitstring/methods.py b/bitstring/methods.py
index e6be89b..18d7acc 100644
--- a/bitstring/methods.py
+++ b/bitstring/methods.py
@@ -1,4 +1,5 @@
from __future__ import annotations
+
import bitstring
from bitstring.bitstream import BitStream
from bitstring.utils import tokenparser
@@ -8,7 +9,7 @@ from bitstring.bitstore import BitStore
from bitstring.bitstore_helpers import bitstore_from_token
-def pack(fmt: Union[str, List[str]], *values, **kwargs) ->BitStream:
+def pack(fmt: Union[str, List[str]], *values, **kwargs) -> BitStream:
"""Pack the values according to the format string and return a new BitStream.
fmt -- A single string or a list of strings with comma separated tokens
@@ -43,4 +44,52 @@ def pack(fmt: Union[str, List[str]], *values, **kwargs) ->BitStream:
>>> u = pack('uint:8=a, uint:8=b, uint:55=a', a=6, b=44)
"""
- pass
+ tokens = []
+ if isinstance(fmt, str):
+ fmt = [fmt]
+ try:
+ for f_item in fmt:
+ _, tkns = tokenparser(f_item, tuple(sorted(kwargs.keys())))
+ tokens.extend(tkns)
+ except ValueError as e:
+ raise CreationError(*e.args)
+ value_iter = iter(values)
+ bsl: List[BitStore] = []
+ try:
+ for name, length, value in tokens:
+ # If the value is in the kwd dictionary then it takes precedence.
+ value = kwargs.get(value, value)
+ # If the length is in the kwd dictionary then use that too.
+ length = kwargs.get(length, length)
+ # Also if we just have a dictionary name then we want to use it
+ if name in kwargs and length is None and value is None:
+ bsl.append(BitStream(kwargs[name])._bitstore)
+ continue
+ if length is not None:
+ length = int(length)
+ if value is None and name != 'pad':
+ # Take the next value from the ones provided
+ value = next(value_iter)
+ if name == 'bits':
+ value = bitstring.bits.Bits(value)
+ if length is not None and length != len(value):
+ raise CreationError(f"Token with length {length} packed with value of length {len(value)}.")
+ bsl.append(value._bitstore)
+ continue
+ bsl.append(bitstore_from_token(name, length, value))
+ except StopIteration:
+ raise CreationError(f"Not enough parameters present to pack according to the "
+ f"format. {len(tokens)} values are needed.")
+
+ try:
+ next(value_iter)
+ except StopIteration:
+ # Good, we've used up all the *values.
+ s = BitStream()
+ if bitstring.options.lsb0:
+ bsl.reverse()
+ for b in bsl:
+ s._bitstore += b
+ return s
+
+ raise CreationError(f"Too many parameters present to pack according to the format. Only {len(tokens)} values were expected.")
diff --git a/bitstring/mxfp.py b/bitstring/mxfp.py
index be90cd1..d675c35 100644
--- a/bitstring/mxfp.py
+++ b/bitstring/mxfp.py
@@ -7,63 +7,200 @@ import zlib
from typing import Optional
+def round_to_nearest_ties_to_even(lut_int_to_float, lower: int, f: float) -> Optional[int]:
+ upper = lower + 1
+ # Special case for LUTs without a negative zero.
+ lower_float = 0.0 if lower == 128 else lut_int_to_float[lower]
+ upper_float = lut_int_to_float[upper]
+ if upper_float < lower_float:
+ lower, upper = upper, lower
+ lower_float, upper_float = upper_float, lower_float
+ if f == lower_float:
+ return lower
+ if f == upper_float:
+ return upper
+ if lower_float < f < upper_float:
+ d1 = f - lower_float
+ d2 = upper_float - f
+ if d1 < d2:
+ return lower
+ if d2 < d1:
+ return upper
+ return lower if lower % 2 == 0 else upper
+ return None
+
+
class MXFPFormat:
"""Defining an MXFP micro-scaling floating point format"""
- def __init__(self, exp_bits: int, mantissa_bits: int, bias: int,
- mxfp_overflow: str):
+ def __init__(self, exp_bits: int, mantissa_bits: int, bias: int, mxfp_overflow: str):
self.exp_bits = exp_bits
self.mantissa_bits = mantissa_bits
self.bias = bias
self.mxfp_overflow = mxfp_overflow
- self.pos_clamp_value = (1 << self.exp_bits + self.mantissa_bits) - 1
- self.neg_clamp_value = (1 << 1 + self.exp_bits + self.mantissa_bits
- ) - 1
+
+ self.pos_clamp_value = (1 << (self.exp_bits + self.mantissa_bits)) - 1
+ self.neg_clamp_value = (1 << (1 + self.exp_bits + self.mantissa_bits)) - 1
+
+ # Special cases for e4m3 and e5m2
if self.exp_bits == 4 and self.mantissa_bits == 3:
if self.mxfp_overflow == 'saturate':
- self.pos_clamp_value = 126
- self.neg_clamp_value = 254
+ self.pos_clamp_value = 0b01111110 # 448
+ self.neg_clamp_value = 0b11111110 # -448
else:
- self.pos_clamp_value = self.neg_clamp_value = 255
+ self.pos_clamp_value = self.neg_clamp_value = 0b11111111 # NaN
if self.exp_bits == 5 and self.mantissa_bits == 2:
if self.mxfp_overflow == 'saturate':
- self.pos_clamp_value = 123
- self.neg_clamp_value = 251
+ self.pos_clamp_value = 0b01111011 # 57344
+ self.neg_clamp_value = 0b11111011 # -57344
else:
- self.pos_clamp_value = 124
- self.neg_clamp_value = 252
+ self.pos_clamp_value = 0b01111100 # +inf
+ self.neg_clamp_value = 0b11111100 # -inf
+
+ # If we calculate these LUTs now it creates a bootstrap problem in generate_luts.py.
self.lut_float16_to_mxfp = None
self.lut_int_to_float = None
def __str__(self):
- return (
- f"MXFPFormat(exp_bits={self.exp_bits}, mantissa_bits={self.mantissa_bits}, bias={self.bias}, mxfp_overflow='{self.mxfp_overflow}')"
- )
+ return f"MXFPFormat(exp_bits={self.exp_bits}, mantissa_bits={self.mantissa_bits}, bias={self.bias}, mxfp_overflow='{self.mxfp_overflow}')"
+
+ def decompress_luts(self):
+ int_to_float_compressed, float16_to_mxfp_compressed = mxfp_luts_compressed[(self.exp_bits, self.mantissa_bits, self.bias, self.mxfp_overflow)]
+ self.lut_float16_to_mxfp = zlib.decompress(float16_to_mxfp_compressed)
+ dec = zlib.decompress(int_to_float_compressed)
+ self.lut_int_to_float = struct.unpack(f'<{len(dec) // 4}f', dec)
- def float_to_int(self, f: float) ->int:
+ def create_luts(self):
+ self.lut_int_to_float = self.createLUT_for_int_to_float()
+ self.lut_float16_to_mxfp = self.createLUT_for_float16_to_mxfp()
+
+ def float_to_int(self, f: float) -> int:
"""Given a Python float convert to the best mxfp float (expressed as an int) that represents it."""
- pass
+ # First convert the float to a float16, then a 16 bit uint
+ try:
+ b = struct.pack('>e', f)
+ except (OverflowError, struct.error):
+ # Return the largest representable positive or negative value
+ return self.pos_clamp_value if f > 0 else self.neg_clamp_value
+
+ f16_int = int.from_bytes(b, byteorder='big')
+ # Then use this as an index to our large LUT
+ return self.lut_float16_to_mxfp[f16_int]
- def createLUT_for_int_to_float(self) ->array.array:
+ def slow_float_to_int(self, f: float) -> int:
+ # Slow, but easier to follow than the faster version.
+ # The output int has the binary sequence needed for the float.
+ length = 1 + self.exp_bits + self.mantissa_bits
+ values = 1 << length
+ # First get the NaN case out of the way
+ if math.isnan(f):
+ if length == 8:
+ return 0xff # Works for both e5m2 and e4m3
+ # For smaller lengths, NaN isn't supported so we instead return an invalid value to detect later
+ return 0xff
+ # This is so we can distinguish between 0.0 and -0.0
+ is_positive = math.copysign(1.0, f) == 1.0
+ if is_positive:
+ # Positive, so top bit is not set
+ for i in range(values // 2 - 1):
+ upper = self.lut_int_to_float[i + 1]
+ if upper == float('inf'):
+ break
+ x = round_to_nearest_ties_to_even(self.lut_int_to_float, i, f)
+ if x is not None:
+ return x
+ return self.pos_clamp_value
+ else:
+ # Negative, so top bit is set
+ for i in range(values // 2, values - 1):
+ lower = self.lut_int_to_float[i + 1]
+ if lower == float('-inf'):
+ break
+ x = round_to_nearest_ties_to_even(self.lut_int_to_float, i, f)
+ if x is not None:
+ return x
+ # Clip to negative max
+ return self.neg_clamp_value
+
+ def createLUT_for_int_to_float(self) -> array.array:
"""Create a LUT to convert an int in representing a MXFP float into a Python float"""
- pass
+ i2f = []
+ length = 1 + self.exp_bits + self.mantissa_bits
+ for i in range(1 << length):
+ b = bitarray.util.int2ba(i, length=length, endian='big', signed=False)
+ sign = b[0]
+ exponent = bitarray.util.ba2int(b[1:1 + self.exp_bits])
+ significand = b[1 + self.exp_bits:]
+ if exponent == 0:
+ significand = bitarray.bitarray('0') + significand
+ exponent = -self.bias + 1
+ else:
+ significand = bitarray.bitarray('1') + significand
+ exponent -= self.bias
+ f = float(bitarray.util.ba2int(significand)) / (2.0 ** self.mantissa_bits)
+ f *= 2 ** exponent
+ if length == 8:
+ # Some special cases
+ if self.exp_bits == 5:
+ if i in [0b01111100, 0b11111100]:
+ f = float('inf')
+ if i in [0b01111101, 0b11111101, 0b01111110, 0b11111110, 0b01111111, 0b11111111]:
+ f = float('nan')
+ if self.exp_bits == 4:
+ if i in [0b01111111, 0b11111111]:
+ f = float('nan')
+ i2f.append(f if not sign else -f)
+ return array.array('f', i2f)
- def createLUT_for_float16_to_mxfp(self) ->bytes:
+ def createLUT_for_float16_to_mxfp(self) -> bytes:
"""Create a LUT to convert a float16 into a MXFP format"""
- pass
-
-
-e2m1mxfp_fmt = MXFPFormat(exp_bits=2, mantissa_bits=1, bias=1,
- mxfp_overflow='saturate')
-e2m3mxfp_fmt = MXFPFormat(exp_bits=2, mantissa_bits=3, bias=1,
- mxfp_overflow='saturate')
-e3m2mxfp_fmt = MXFPFormat(exp_bits=3, mantissa_bits=2, bias=3,
- mxfp_overflow='saturate')
-e4m3mxfp_saturate_fmt = MXFPFormat(exp_bits=4, mantissa_bits=3, bias=7,
- mxfp_overflow='saturate')
-e5m2mxfp_saturate_fmt = MXFPFormat(exp_bits=5, mantissa_bits=2, bias=15,
- mxfp_overflow='saturate')
-e4m3mxfp_overflow_fmt = MXFPFormat(exp_bits=4, mantissa_bits=3, bias=7,
- mxfp_overflow='overflow')
-e5m2mxfp_overflow_fmt = MXFPFormat(exp_bits=5, mantissa_bits=2, bias=15,
- mxfp_overflow='overflow')
+ # Used to create the LUT that was compressed and stored for the fp8 code
+ length = 1 + self.exp_bits + self.mantissa_bits
+ if length == 8:
+ import gfloat
+ from gfloat.formats import format_info_ocp_e5m2, format_info_ocp_e4m3
+ fi = format_info_ocp_e5m2 if self.exp_bits == 5 else format_info_ocp_e4m3
+
+ fp16_to_fp8 = bytearray(1 << 16)
+ for i in range(1 << 16):
+ b = struct.pack('>H', i)
+ f, = struct.unpack('>e', b)
+ fp = gfloat.round_float(fi, f, sat=self.mxfp_overflow == 'saturate')
+ if math.isnan(fp):
+ fp8_i = 0b11111111
+ else:
+ # Special case for negative zero
+ if fp == 0.0 and math.copysign(1.0, fp) == -1.0:
+ fp8_i = 0b10000000
+ else:
+ fp8_i = self.lut_int_to_float.index(fp)
+ fp16_to_fp8[i] = fp8_i
+ return bytes(fp16_to_fp8)
+ else:
+ assert length in [4, 6]
+ fp16_to_fp8 = bytearray(1 << 16)
+ for i in range(1 << 16):
+ b = struct.pack('>H', i)
+ f, = struct.unpack('>e', b)
+ fp8_i = self.slow_float_to_int(f)
+ fp16_to_fp8[i] = fp8_i
+ return bytes(fp16_to_fp8)
+
+
+e2m1mxfp_fmt = MXFPFormat(exp_bits=2, mantissa_bits=1, bias=1, mxfp_overflow='saturate')
+e2m3mxfp_fmt = MXFPFormat(exp_bits=2, mantissa_bits=3, bias=1, mxfp_overflow='saturate')
+e3m2mxfp_fmt = MXFPFormat(exp_bits=3, mantissa_bits=2, bias=3, mxfp_overflow='saturate')
+e4m3mxfp_saturate_fmt = MXFPFormat(exp_bits=4, mantissa_bits=3, bias=7, mxfp_overflow='saturate')
+e5m2mxfp_saturate_fmt = MXFPFormat(exp_bits=5, mantissa_bits=2, bias=15, mxfp_overflow='saturate')
+e4m3mxfp_overflow_fmt = MXFPFormat(exp_bits=4, mantissa_bits=3, bias=7, mxfp_overflow='overflow')
+e5m2mxfp_overflow_fmt = MXFPFormat(exp_bits=5, mantissa_bits=2, bias=15, mxfp_overflow='overflow')
+
+
+def decompress_luts():
+ e2m1mxfp_fmt.decompress_luts()
+ e2m3mxfp_fmt.decompress_luts()
+ e3m2mxfp_fmt.decompress_luts()
+ e4m3mxfp_saturate_fmt.decompress_luts()
+ e5m2mxfp_saturate_fmt.decompress_luts()
+ e4m3mxfp_overflow_fmt.decompress_luts()
+ e5m2mxfp_overflow_fmt.decompress_luts()
diff --git a/bitstring/utils.py b/bitstring/utils.py
index 623d69b..4dae4d9 100644
--- a/bitstring/utils.py
+++ b/bitstring/utils.py
@@ -1,47 +1,171 @@
from __future__ import annotations
+
import functools
import re
from typing import Tuple, List, Optional, Pattern, Dict, Union, Match
-NAME_INT_RE: Pattern[str] = re.compile('^([a-zA-Z][a-zA-Z0-9_]*?):?(\\d*)$')
-NAME_KWARG_RE: Pattern[str] = re.compile(
- '^([a-zA-Z][a-zA-Z0-9_]*?):?([a-zA-Z0-9_]+)$')
+
+
+# A token name followed by optional : then an integer number
+NAME_INT_RE: Pattern[str] = re.compile(r'^([a-zA-Z][a-zA-Z0-9_]*?):?(\d*)$')
+
+# A token name followed by optional : then an arbitrary keyword
+NAME_KWARG_RE: Pattern[str] = re.compile(r'^([a-zA-Z][a-zA-Z0-9_]*?):?([a-zA-Z0-9_]+)$')
+
CACHE_SIZE = 256
-DEFAULT_BITS: Pattern[str] = re.compile('^(?P<len>[^=]+)?(=(?P<value>.*))?$',
- re.IGNORECASE)
-MULTIPLICATIVE_RE: Pattern[str] = re.compile('^(?P<factor>.*)\\*(?P<token>.+)')
-LITERAL_RE: Pattern[str] = re.compile('^(?P<name>0([xob]))(?P<value>.+)',
- re.IGNORECASE)
-STRUCT_PACK_RE: Pattern[str] = re.compile(
- '^(?P<endian>[<>@=])(?P<fmt>(?:\\d*[bBhHlLqQefd])+)$')
-BYTESWAP_STRUCT_PACK_RE: Pattern[str] = re.compile(
- '^(?P<endian>[<>@=])?(?P<fmt>(?:\\d*[bBhHlLqQefd])+)$')
-SINGLE_STRUCT_PACK_RE: Pattern[str] = re.compile(
- '^(?P<endian>[<>@=])(?P<fmt>[bBhHlLqQefd])$')
-STRUCT_SPLIT_RE: Pattern[str] = re.compile('\\d*[bBhHlLqQefd]')
-REPLACEMENTS_BE: Dict[str, str] = {'b': 'int8', 'B': 'uint8', 'h':
- 'intbe16', 'H': 'uintbe16', 'l': 'intbe32', 'L': 'uintbe32', 'q':
- 'intbe64', 'Q': 'uintbe64', 'e': 'floatbe16', 'f': 'floatbe32', 'd':
- 'floatbe64'}
-REPLACEMENTS_LE: Dict[str, str] = {'b': 'int8', 'B': 'uint8', 'h':
- 'intle16', 'H': 'uintle16', 'l': 'intle32', 'L': 'uintle32', 'q':
- 'intle64', 'Q': 'uintle64', 'e': 'floatle16', 'f': 'floatle32', 'd':
- 'floatle64'}
-REPLACEMENTS_NE: Dict[str, str] = {'b': 'int8', 'B': 'uint8', 'h':
- 'intne16', 'H': 'uintne16', 'l': 'intne32', 'L': 'uintne32', 'q':
- 'intne64', 'Q': 'uintne64', 'e': 'floatne16', 'f': 'floatne32', 'd':
- 'floatne64'}
-PACK_CODE_SIZE: Dict[str, int] = {'b': 1, 'B': 1, 'h': 2, 'H': 2, 'l': 4,
- 'L': 4, 'q': 8, 'Q': 8, 'e': 2, 'f': 4, 'd': 8}
-
-
-def structparser(m: Match[str]) ->List[str]:
+
+DEFAULT_BITS: Pattern[str] = re.compile(r'^(?P<len>[^=]+)?(=(?P<value>.*))?$', re.IGNORECASE)
+
+MULTIPLICATIVE_RE: Pattern[str] = re.compile(r'^(?P<factor>.*)\*(?P<token>.+)')
+
+# Hex, oct or binary literals
+LITERAL_RE: Pattern[str] = re.compile(r'^(?P<name>0([xob]))(?P<value>.+)', re.IGNORECASE)
+
+# An endianness indicator followed by one or more struct.pack codes
+STRUCT_PACK_RE: Pattern[str] = re.compile(r'^(?P<endian>[<>@=])(?P<fmt>(?:\d*[bBhHlLqQefd])+)$')
+# The same as above, but it doesn't insist on an endianness as it's byteswapping anyway.
+BYTESWAP_STRUCT_PACK_RE: Pattern[str] = re.compile(r'^(?P<endian>[<>@=])?(?P<fmt>(?:\d*[bBhHlLqQefd])+)$')
+# An endianness indicator followed by exactly one struct.pack codes
+SINGLE_STRUCT_PACK_RE: Pattern[str] = re.compile(r'^(?P<endian>[<>@=])(?P<fmt>[bBhHlLqQefd])$')
+
+# A number followed by a single character struct.pack code
+STRUCT_SPLIT_RE: Pattern[str] = re.compile(r'\d*[bBhHlLqQefd]')
+
+# These replicate the struct.pack codes
+# Big-endian
+REPLACEMENTS_BE: Dict[str, str] = {'b': 'int8', 'B': 'uint8',
+ 'h': 'intbe16', 'H': 'uintbe16',
+ 'l': 'intbe32', 'L': 'uintbe32',
+ 'q': 'intbe64', 'Q': 'uintbe64',
+ 'e': 'floatbe16', 'f': 'floatbe32', 'd': 'floatbe64'}
+# Little-endian
+REPLACEMENTS_LE: Dict[str, str] = {'b': 'int8', 'B': 'uint8',
+ 'h': 'intle16', 'H': 'uintle16',
+ 'l': 'intle32', 'L': 'uintle32',
+ 'q': 'intle64', 'Q': 'uintle64',
+ 'e': 'floatle16', 'f': 'floatle32', 'd': 'floatle64'}
+
+# Native-endian
+REPLACEMENTS_NE: Dict[str, str] = {'b': 'int8', 'B': 'uint8',
+ 'h': 'intne16', 'H': 'uintne16',
+ 'l': 'intne32', 'L': 'uintne32',
+ 'q': 'intne64', 'Q': 'uintne64',
+ 'e': 'floatne16', 'f': 'floatne32', 'd': 'floatne64'}
+
+# Size in bytes of all the pack codes.
+PACK_CODE_SIZE: Dict[str, int] = {'b': 1, 'B': 1, 'h': 2, 'H': 2, 'l': 4, 'L': 4,
+ 'q': 8, 'Q': 8, 'e': 2, 'f': 4, 'd': 8}
+
+
+def structparser(m: Match[str]) -> List[str]:
"""Parse struct-like format string token into sub-token list."""
- pass
+ endian = m.group('endian')
+ # Split the format string into a list of 'q', '4h' etc.
+ formatlist = re.findall(STRUCT_SPLIT_RE, m.group('fmt'))
+ # Now deal with multiplicative factors, 4h -> hhhh etc.
+ fmt = ''.join([f[-1] * int(f[:-1]) if len(f) != 1 else
+ f for f in formatlist])
+ if endian in '@=':
+ # Native endianness
+ tokens = [REPLACEMENTS_NE[c] for c in fmt]
+ elif endian == '<':
+ tokens = [REPLACEMENTS_LE[c] for c in fmt]
+ else:
+ assert endian == '>'
+ tokens = [REPLACEMENTS_BE[c] for c in fmt]
+ return tokens
+
+
+@functools.lru_cache(CACHE_SIZE)
+def parse_name_length_token(fmt: str, **kwargs) -> Tuple[str, Optional[int]]:
+ # Any single token with just a name and length
+ if m2 := NAME_INT_RE.match(fmt):
+ name = m2.group(1)
+ length_str = m2.group(2)
+ length = None if length_str == '' else int(length_str)
+ else:
+ # Maybe the length is in the kwargs?
+ if m := NAME_KWARG_RE.match(fmt):
+ name = m.group(1)
+ try:
+ length_str = kwargs[m.group(2)]
+ except KeyError:
+ raise ValueError(f"Can't parse 'name[:]length' token '{fmt}'.")
+ length = int(length_str)
+ else:
+ raise ValueError(f"Can't parse 'name[:]length' token '{fmt}'.")
+ return name, length
+
+
+@functools.lru_cache(CACHE_SIZE)
+def parse_single_struct_token(fmt: str) -> Optional[Tuple[str, Optional[int]]]:
+ if m := SINGLE_STRUCT_PACK_RE.match(fmt):
+ endian = m.group('endian')
+ f = m.group('fmt')
+ if endian == '>':
+ fmt = REPLACEMENTS_BE[f]
+ elif endian == '<':
+ fmt = REPLACEMENTS_LE[f]
+ else:
+ assert endian in '=@'
+ fmt = REPLACEMENTS_NE[f]
+ return parse_name_length_token(fmt)
+ else:
+ return None
+
+
+@functools.lru_cache(CACHE_SIZE)
+def parse_single_token(token: str) -> Tuple[str, str, Optional[str]]:
+ if (equals_pos := token.find('=')) == -1:
+ value = None
+ else:
+ value = token[equals_pos + 1:]
+ token = token[:equals_pos]
+
+ if m2 := NAME_INT_RE.match(token):
+ name = m2.group(1)
+ length_str = m2.group(2)
+ length = None if length_str == '' else length_str
+ elif m3 := NAME_KWARG_RE.match(token):
+ # name then a keyword for a length
+ name = m3.group(1)
+ length = m3.group(2)
+ else:
+ # If you don't specify a 'name' then the default is 'bits'
+ name = 'bits'
+ length = token
+ return name, length, value
+
+
+@functools.lru_cache(CACHE_SIZE)
+def preprocess_tokens(fmt: str) -> List[str]:
+ # Remove whitespace and expand brackets
+ fmt = expand_brackets(''.join(fmt.split()))
+
+ # Split tokens by ',' and remove whitespace
+ # The meta_tokens can either be ordinary single tokens or multiple struct-format token strings.
+ meta_tokens = [f.strip() for f in fmt.split(',')]
+ final_tokens = []
+
+ for meta_token in meta_tokens:
+ if meta_token == '':
+ continue
+ # Extract factor and actual token if a multiplicative factor exists
+ factor = 1
+ if m := MULTIPLICATIVE_RE.match(meta_token):
+ factor = int(m.group('factor'))
+ meta_token = m.group('token')
+
+ # Parse struct-like format into sub-tokens or treat as single token
+ tokens = structparser(m) if (m := STRUCT_PACK_RE.match(meta_token)) else [meta_token]
+
+ # Extend final tokens list with parsed tokens, repeated by the factor
+ final_tokens.extend(tokens * factor)
+ return final_tokens
@functools.lru_cache(CACHE_SIZE)
-def tokenparser(fmt: str, keys: Tuple[str, ...]=()) ->Tuple[bool, List[
- Tuple[str, Union[int, str, None], Optional[str]]]]:
+def tokenparser(fmt: str, keys: Tuple[str, ...] = ()) -> \
+ Tuple[bool, List[Tuple[str, Union[int, str, None], Optional[str]]]]:
"""Divide the format string into tokens and parse them.
Return stretchy token and list of [initialiser, length, value]
@@ -54,12 +178,61 @@ def tokenparser(fmt: str, keys: Tuple[str, ...]=()) ->Tuple[bool, List[
tokens must be of the form: [factor*][initialiser][:][length][=value]
"""
- pass
+ tokens = preprocess_tokens(fmt)
+ stretchy_token = False
+ ret_vals: List[Tuple[str, Union[str, int, None], Optional[str]]] = []
+ for token in tokens:
+ if keys and token in keys:
+ # Don't bother parsing it, it's a keyword argument
+ ret_vals.append((token, None, None))
+ continue
+ if token == '':
+ continue
+ # Match literal tokens of the form 0x... 0o... and 0b...
+ if m := LITERAL_RE.match(token):
+ ret_vals.append((m.group('name'), None, m.group('value')))
+ continue
+ name, length, value = parse_single_token(token)
+ if length is None:
+ stretchy_token = True
+ if length is not None:
+ # Try converting length to int, otherwise check it's a key.
+ try:
+ length = int(length)
+ except ValueError:
+ if not keys or length not in keys:
+ raise ValueError(f"Don't understand length '{length}' of token.")
+ ret_vals.append((name, length, value))
+ return stretchy_token, ret_vals
-BRACKET_RE = re.compile('(?P<factor>\\d+)\\*\\(')
+BRACKET_RE = re.compile(r'(?P<factor>\d+)\*\(')
-def expand_brackets(s: str) ->str:
+def expand_brackets(s: str) -> str:
"""Expand all brackets."""
- pass
+ while True:
+ start = s.find('(')
+ if start == -1:
+ break
+ count = 1 # Number of hanging open brackets
+ p = start + 1
+ while p < len(s):
+ count += (s[p] == '(') - (s[p] == ')')
+ if count == 0:
+ break
+ p += 1
+ if count != 0:
+ raise ValueError(f"Unbalanced parenthesis in '{s}'.")
+ if start == 0 or s[start - 1] != '*':
+ s = s[0:start] + s[start + 1:p] + s[p + 1:]
+ else:
+ # Looks for first number*(
+ m = BRACKET_RE.search(s)
+ if m:
+ factor = int(m.group('factor'))
+ matchstart = m.start('factor')
+ s = s[0:matchstart] + (factor - 1) * (s[start + 1:p] + ',') + s[start + 1:p] + s[p + 1:]
+ else:
+ raise ValueError(f"Failed to parse '{s}'.")
+ return s