データをchunkして、encodeして、セグメントに分割して、nodeに保存する
import math
from reedsolo import RSCodec
import random
SEGMENT_SIZE = 8
DATA_SEGMENTS = 4
TOTAL_SEGMENTS = 6
CHUNK_SIZE = SEGMENT_SIZE * DATA_SEGMENTS
rsc = RSCodec(TOTAL_SEGMENTS - DATA_SEGMENTS)
def generate_fake_file(size=40):
return bytearray([random.randint(0, 255) for _ in range(size)])
def split_into_chunks(data, chunk_size):
return [data[i:i+chunk_size] for i in range(0, len(data), chunk_size)]
def encode_chunk(chunk):
segments = [chunk[i:i+SEGMENT_SIZE] for i in range(0, len(chunk), SEGMENT_SIZE)]
joined_data = b''.join(segments)
encoded = rsc.encode(joined_data)
seg_len = len(encoded) // TOTAL_SEGMENTS
encoded_segments = [encoded[i:i+seg_len] for i in range(0, len(encoded), seg_len)]
return encoded_segments
def distribute_segments(segments):
nodes = {}
for i, seg in enumerate(segments):
nodes[f"node_{i}"] = seg
return nodes
fake_file = generate_fake_file(40)
print(fake_file)
chunks = split_into_chunks(fake_file, CHUNK_SIZE)
for i, chunk in enumerate(chunks):
print(f"チャンク {i+1}: {chunk}")
encoded_segments = encode_chunk(chunk)
print(f" ↳ 符号化セグメント数: {len(encoded_segments)} ")
distributed = distribute_segments(encoded_segments)
for node, seg in distributed.items():
print(f" {node}に保存: {seg}")
print()
$ python3 sia.py
bytearray(b”Xz\xe12\x1a\xe7aq\xf3\’\x11\xcb\xa7\xf0\x0f*r\xd5Y4y#|\xb2\xe0\xeb\xb4\x14+\xedl\xe2\xf9\x1b\xe9s\x93\xc0\x94\xde”)
チャンク 1: bytearray(b”Xz\xe12\x1a\xe7aq\xf3\’\x11\xcb\xa7\xf0\x0f*r\xd5Y4y#|\xb2\xe0\xeb\xb4\x14+\xedl\xe2″)
↳ 符号化セグメント数: 7
node_0に保存: bytearray(b’Xz\xe12\x1a’)
node_1に保存: bytearray(b”\xe7aq\xf3\'”)
node_2に保存: bytearray(b’\x11\xcb\xa7\xf0\x0f’)
node_3に保存: bytearray(b’*r\xd5Y4′)
node_4に保存: bytearray(b’y#|\xb2\xe0′)
node_5に保存: bytearray(b’\xeb\xb4\x14+\xed’)
node_6に保存: bytearray(b’l\xe2q\xac’)
チャンク 2: bytearray(b’\xf9\x1b\xe9s\x93\xc0\x94\xde’)
↳ 符号化セグメント数: 10
node_0に保存: bytearray(b’\xf9′)
node_1に保存: bytearray(b’\x1b’)
node_2に保存: bytearray(b’\xe9′)
node_3に保存: bytearray(b’s’)
node_4に保存: bytearray(b’\x93′)
node_5に保存: bytearray(b’\xc0′)
node_6に保存: bytearray(b’\x94′)
node_7に保存: bytearray(b’\xde’)
node_8に保存: bytearray(b’\xb9′)
node_9に保存: bytearray(b’\xd8′)
