Skip to content

Commit c1c0fbc

Browse files
RobSmithDevRobSmithDev
committed
Updates. Contains LZ4 compress/decompress but it doesnt work
1 parent 0563921 commit c1c0fbc

6 files changed

Lines changed: 233 additions & 35 deletions

File tree

‎NetDevice/device.c‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -753,15 +753,12 @@ __saveds void frame_proc() {
753753
timeLastWifiCheck.tv_secs = timeWifiCheck.tv_secs;
754754
}
755755

756-
UBYTE morePackets = 0;
757756
if (db->db_online) {
758757
USHORT counter = 0;
759758

760-
recv = SetSignal(0L, 0L);
761-
759+
recv = SetSignal(0L, 0L);
762760
if (!(recv & SIGBREAKF_CTRL_C)) {
763-
morePackets = paulaNetReceivePackets(device, db, (PacketReceivedCallback)onPacketReceived);
764-
if (morePackets == DEVICESTATUS_ERROR) D(("ERROROORORO"));
761+
paulaNetReceivePackets(device, db, (PacketReceivedCallback)onPacketReceived);
765762
}
766763

767764
// Transmit

‎NetDevice/paulanetio.c‎

Lines changed: 84 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,14 @@ _____ _ _ _ ___________
2626
#include <stdio.h>
2727
#include <stdlib.h>
2828

29-
// #define DEBUGGING 1
3029
#include "debug.h"
3130

3231
typedef UBYTE DataMode;
3332
#define dmAccessPoints 1
3433
#define dmConfig 2
3534
#define dmEthernet 3
3635

37-
#define PAULA_INDEX_PADDING 128
36+
#define PAULA_INDEX_PADDING 64
3837
#define PAULA_WRITE_PADDING (4+2) // (write padding in bytes, 4 for the two sync)
3938
#define CHIPBUFFER_SIZE (MAX_TRANSFER_SIZE + (PAULA_INDEX_PADDING<<2))
4039

@@ -48,6 +47,8 @@ typedef UBYTE DataMode;
4847

4948
#define COMPRESSION_WORSTCASE (((ETH_MTU+127)/127)+2)
5049

50+
#define LZ4_MIN_MATCH 4
51+
5152
#define TRACK_OFFSET(track, side) ((track<<1) + side)
5253
#define TRACK_SEEK_OFFSET(track, side) (((track<<1) + side) * NUMSECS * TD_SECTOR)
5354

@@ -100,33 +101,51 @@ struct CompressionHeader {
100101
UWORD originalSize;
101102
};
102103

104+
static const UWORD stuffChar[] = {0x16c0, 0x4e75};
105+
#ifdef DEBUG
106+
void ssprintf(struct PaulaNET* dev, const char *messageFormat, char* buf, ...) {
107+
va_list args;
108+
va_start(args, buf); /* FIX: last named param before ... is buf, not messageFormat */
109+
RawDoFmt((STRPTR)messageFormat, (APTR)args, (void (*)(void))stuffChar, buf);
110+
va_end(args);
111+
}
103112

104-
#ifdef DEBUGGING
105-
void hexDump(BPTR out, const void* data, ULONG length) {
113+
void hexDump(struct PaulaNET* out, const void* data, ULONG length) {
106114
const UBYTE* p = (const UBYTE*)data;
115+
char buffer[160]; /* slightly larger to be safe */
116+
char tmp[32];
107117
ULONG i, j;
108118
for (i = 0; i < length; i += 16) {
109-
FPrintf(out, "%04lx: ", i);
119+
buffer[0] = '\0';
120+
ssprintf(out, "%04lx: ", tmp, i);
121+
strcat(buffer, tmp);
110122
for (j = 0; j < 16; j++) {
111-
if (i + j < length)
112-
FPrintf(out, "%02lx ", (ULONG)p[i+j]);
113-
else
114-
FPrintf(out, " ");
123+
if (i + j < length) {
124+
ssprintf(out, "%02lx ", tmp, (ULONG)p[i+j]);
125+
strcat(buffer, tmp);
126+
} else strcat(buffer, " ");
115127
}
116-
FPrintf(out, " ");
128+
129+
strcat(buffer, " ");
117130
for (j = 0; j < 16 && i + j < length; j++) {
118131
UBYTE c = p[i+j];
119-
FPrintf(out, "%lc", (ULONG)(c >= 32 && c < 127 ? c : '.'));
132+
tmp[0] = (c >= 32 && c < 127) ? (char)c : '.';
133+
tmp[1] = '\0';
134+
strcat(buffer, tmp);
120135
}
121-
FPrintf(out, "\n");
136+
137+
strcat(buffer, "\n");
138+
D((buffer));
122139
}
123140
}
124141
#endif
125142

143+
144+
126145
// Formatted logging to console window
127-
static const UWORD stuffChar[] = {0x16c0, 0x4e75};
146+
128147
static void logMessagef(struct PaulaNET* dev, const char *messageFormat, ...) {
129-
#ifdef DEBUGGING
148+
#ifdef DEBUG
130149
char buf[100];
131150
va_list args;
132151
va_start(args, messageFormat);
@@ -241,6 +260,55 @@ UWORD rleDecompress(struct PaulaNET* dev, const UBYTE* src, UWORD srclen, UBYTE*
241260
return dstPos;
242261
}
243262

263+
// Decompress LZ4 block format data.
264+
UWORD lz4Decompress(struct PaulaNET* dev, const UBYTE* src, UWORD srcLen, UBYTE* dst, UWORD dstMax) {
265+
UWORD srcPos = 0;
266+
UWORD dstPos = 0;
267+
268+
while (srcPos < srcLen) {
269+
// --- Token ---
270+
const UBYTE token = src[srcPos++];
271+
UWORD litLen = token >> 4;
272+
UWORD matchLen = token & 0x0F;
273+
274+
// --- Extra literal length ---
275+
if (litLen == 15) {
276+
UBYTE extra;
277+
do { extra = src[srcPos++]; litLen += extra; } while (extra == 255);
278+
}
279+
280+
// --- Literals ---
281+
if (dstPos + litLen > dstMax) break;
282+
if (srcPos + litLen > srcLen) break;
283+
memcpy(&dst[dstPos], &src[srcPos], litLen);
284+
srcPos += litLen;
285+
dstPos += litLen;
286+
287+
// --- End of stream: final sequence has no match ---
288+
if (srcPos >= srcLen) break;
289+
290+
// --- Match offset ---
291+
if (srcPos + 2 > srcLen) break;
292+
const UWORD offset = ((UWORD)src[srcPos] << 8) | ((UWORD)src[srcPos + 1]);
293+
srcPos += 2;
294+
if (offset == 0 || offset > dstPos) break; // corrupt
295+
296+
// --- Extra match length ---
297+
matchLen += LZ4_MIN_MATCH;
298+
if ((token & 0x0F) == 15) {
299+
UBYTE extra;
300+
do { extra = src[srcPos++]; matchLen += extra; } while (extra == 255);
301+
}
302+
303+
// --- Match copy (may overlap, so byte-by-byte) ---
304+
if (dstPos + matchLen > dstMax) break;
305+
const UBYTE* matchSrc = &dst[dstPos - offset];
306+
for (UWORD i = 0; i < matchLen; i++) dst[dstPos++] = matchSrc[i];
307+
}
308+
309+
return dstPos;
310+
}
311+
244312
// This generate a valid MFM output for the data input, it's not how it should be done (bits being spread into bytes, its an odd/even approach, which is faster)
245313
int mfmEncode(const UBYTE* src, UWORD srcLen, UWORD* dst, UWORD* lastBit) {
246314
UWORD* out = dst;
@@ -898,9 +966,6 @@ BOOL paulaNetReceivePackets(APTR device, APTR userContext, PacketReceivedCallbac
898966
int realDataLength = dataLength;
899967
if (realDataLength&1) realDataLength++; // has to be even
900968
if (((UBYTE*)pkt) + realDataLength > (UBYTE*)pktEnd) {
901-
#ifdef DEBUGGING
902-
FPrintf(Output(), "Packet data content past end of buffer\n");
903-
#endif
904969
return TRUE;
905970
}
906971

@@ -911,12 +976,11 @@ BOOL paulaNetReceivePackets(APTR device, APTR userContext, PacketReceivedCallbac
911976
callback(userContext, (UBYTE*)dev->compressionSpace, cHeader.originalSize);
912977
} else {
913978
// Compressed
979+
//int res = lz4Decompress(dev, (UBYTE*)pkt, dataLength, (UBYTE*)dev->compressionSpace, CHIPBUFFER_SIZE);
980+
//if (res == cHeader.originalSize) {
914981
if (rleDecompress(dev, (UBYTE*)pkt, dataLength, (UBYTE*)dev->compressionSpace, CHIPBUFFER_SIZE) == cHeader.originalSize) {
915982
callback(userContext, (UBYTE*)dev->compressionSpace, cHeader.originalSize);
916983
} else {
917-
#ifdef DEBUGGING
918-
FPrintf(Output(), "Packet decompressed to wrong size\n");
919-
#endif
920984
}
921985
}
922986

‎Pico/PaulaNET/PaulaNET.cpp‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,10 @@ _____ _ _ _ ___________
3636
/*
3737
PaulaNET Protocol is:
3838
39-
Track 0..79: Normal Amiga Disk
40-
Track 80: Access Points
41-
Track 81: Config
42-
Track 82: Ethernet
39+
Track 0..74, 78+: Normal Amiga Disk
40+
Track 75: Access Points
41+
Track 76: Config
42+
Track 77: Ethernet
4343
4444
Head Selection:
4545
Lower Side: Receive data from Amiga (Core1) Amiga -> Pico
@@ -1303,7 +1303,7 @@ int main() {
13031303
handleWIFI();
13041304
cyw43_arch_gpio_put(CYW43_WL_GPIO_LED_PIN, !isMotorActive); // LED onboard goes off when motor is active
13051305
#ifdef DEBUGGING
1306-
static int lastTrack = -1;
1306+
/* static int lastTrack = -1;
13071307
if (lastTrack != track) {
13081308
lastTrack = track;
13091309
switch (paulaMode) {
@@ -1312,9 +1312,9 @@ int main() {
13121312
case dmEthernet:safePrintf("Track Changed to: %i ETH\n", lastTrack );break;
13131313
case pmDisk:safePrintf("Track Changed to: %i DISK\n", lastTrack );break;
13141314
default: safePrintf("Track Changed to: %i ??\n", track );break;
1315-
}
1316-
1315+
}
13171316
}
1317+
*/
13181318
#endif
13191319

13201320
// Wifi scan
@@ -1394,8 +1394,10 @@ int main() {
13941394
uint8_t* outputPosition = &compressedRxQueue.packets[compressedRxQueue.writeIndex].data[sizeof(CompressionHeader) * 2];
13951395
// Prep compress and encode each packet for transmission
13961396
if (settings.flags & FLAGS_PICO_COMPRESSION) {
1397+
// Experimental LZ4 compressor
1398+
//compressedRxQueue.packets[compressedRxQueue.writeIndex].length = lz4Compress(rxQueue.packets[rxQueue.readIndex].data, rxQueue.packets[rxQueue.readIndex].length, outputPosition, true, (uint16_t*)Core0RunBuffer);
13971399
compressedRxQueue.packets[compressedRxQueue.writeIndex].length = rleCompress(rxQueue.packets[rxQueue.readIndex].data, rxQueue.packets[rxQueue.readIndex].length, outputPosition, true, Core0RunBuffer);
1398-
hdr.compressedSize = SWAP16(compressedRxQueue.packets[compressedRxQueue.writeIndex].length);
1400+
hdr.compressedSize = SWAP16(compressedRxQueue.packets[compressedRxQueue.writeIndex].length);
13991401
// WORD padding
14001402
if (compressedRxQueue.packets[compressedRxQueue.writeIndex].length&1) {
14011403
*(outputPosition+compressedRxQueue.packets[compressedRxQueue.writeIndex].length) = 0;

‎Pico/PaulaNET/rleCompression.cpp‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,4 +127,136 @@ uint16_t rleCalcDecompressSize(const uint8_t* src, uint16_t srclen) {
127127
dstSize += count;
128128
}
129129
return dstSize;
130+
}
131+
132+
133+
134+
// Experimental LZ4 compression Pico->Amiga direction only
135+
136+
// LZ4 constants
137+
#define LZ4_HASH_BITS 10
138+
#define LZ4_HASH_SIZE (1 << LZ4_HASH_BITS) // 4096 entries
139+
#define LZ4_HASH_MASK (LZ4_HASH_SIZE - 1)
140+
#define LZ4_MIN_MATCH 4
141+
#define LZ4_MAX_OFFSET 65535
142+
#define LZ4_LAST_LITERALS 5 // LZ4 spec: last 5 bytes must be literals
143+
144+
// Hash 4 bytes to a table index
145+
static inline uint16_t lz4Hash(uint32_t v) {
146+
return (uint16_t)(((v * 2654435761UL) >> 20) & LZ4_HASH_MASK);
147+
}
148+
149+
// Read 4 bytes as uint32 (unaligned safe)
150+
static inline uint32_t read32(const uint8_t* p) {
151+
uint32_t v;
152+
memcpy(&v, p, 4);
153+
return v;
154+
}
155+
156+
157+
// Emit a literal+match sequence into dst, returns new dstPos
158+
// literals - pointer to start of literal bytes in src
159+
// litLen - number of literal bytes
160+
// matchOffset - backwards offset (1-based), 0 means end-of-stream (no match)
161+
// matchLen - match length (actual bytes, >= LZ4_MIN_MATCH), 0 if no match
162+
static uint16_t emitSequence(uint8_t* dst, uint16_t dstPos, const uint8_t* literals, uint16_t litLen, uint16_t matchOffset, uint16_t matchLen) {
163+
// --- Token byte ---
164+
uint8_t litNibble = (litLen >= 15) ? 15 : (uint8_t)litLen;
165+
uint8_t matchNibble = (matchLen == 0) ? 0 : ((matchLen - LZ4_MIN_MATCH) >= 15) ? 15 : (uint8_t)(matchLen - LZ4_MIN_MATCH);
166+
dst[dstPos++] = (litNibble << 4) | matchNibble;
167+
168+
// --- Extra literal length bytes ---
169+
if (litLen >= 15) {
170+
uint16_t extra = litLen - 15;
171+
while (extra >= 255) { dst[dstPos++] = 255; extra -= 255; }
172+
dst[dstPos++] = (uint8_t)extra;
173+
}
174+
175+
// --- Literal bytes ---
176+
memcpy(&dst[dstPos], literals, litLen);
177+
dstPos += litLen;
178+
179+
// --- Match offset and extra length (omitted on final sequence) ---
180+
if (matchLen > 0) {
181+
dst[dstPos++] = (uint8_t)(matchOffset >> 8);
182+
dst[dstPos++] = (uint8_t)(matchOffset & 0xFF);
183+
184+
if (matchLen - LZ4_MIN_MATCH >= 15) {
185+
uint16_t extra = matchLen - LZ4_MIN_MATCH - 15;
186+
while (extra >= 255) { dst[dstPos++] = 255; extra -= 255; }
187+
dst[dstPos++] = (uint8_t)extra;
188+
}
189+
}
190+
191+
return dstPos;
192+
}
193+
194+
// Compress src into dst using LZ4 block format.
195+
// hashBuffer must be LZ4_HASH_SIZE * sizeof(uint16_t) bytes (2KB).
196+
// Returns compressed size, or 0 on failure (dst too small).
197+
// wordSwap byte-swaps output words, matching your RLE option.
198+
size_t lz4Compress(const uint8_t* src, uint16_t srcLen, uint8_t* dst, bool wordSwap, uint16_t* hashBuffer) {
199+
// Clear hash table (positions default to 0 = unset, we use 0xFFFF as invalid)
200+
memset(hashBuffer, 0xFF, LZ4_HASH_SIZE * sizeof(uint16_t));
201+
202+
uint16_t srcPos = 0; // current input position
203+
uint16_t litStart = 0; // start of current literal run
204+
uint16_t dstPos = 0;
205+
206+
// We must leave LZ4_LAST_LITERALS bytes at the end as literals
207+
const uint16_t matchLimit = (srcLen > LZ4_LAST_LITERALS) ? srcLen - LZ4_LAST_LITERALS : 0;
208+
209+
while (srcPos < matchLimit) {
210+
if (srcPos + LZ4_MIN_MATCH > srcLen) break;
211+
212+
uint32_t v = read32(&src[srcPos]);
213+
uint16_t h = lz4Hash(v);
214+
uint16_t ref = hashBuffer[h];
215+
hashBuffer[h] = srcPos;
216+
217+
// Check for a valid match
218+
bool matched = false;
219+
if (ref != 0xFFFF && srcPos - ref <= LZ4_MAX_OFFSET && ref < srcPos) {
220+
if (read32(&src[ref]) == v) {
221+
// Extend match forward
222+
uint16_t matchLen = LZ4_MIN_MATCH;
223+
uint16_t maxMatch = srcLen - srcPos;
224+
if (maxMatch > LZ4_LAST_LITERALS) maxMatch -= LZ4_LAST_LITERALS;
225+
while (matchLen < maxMatch && src[srcPos + matchLen] == src[ref + matchLen])
226+
matchLen++;
227+
228+
uint16_t litLen = srcPos - litStart;
229+
uint16_t offset = srcPos - ref;
230+
231+
dstPos = emitSequence(dst, dstPos, &src[litStart], litLen, offset, matchLen);
232+
233+
srcPos += matchLen;
234+
litStart = srcPos;
235+
matched = true;
236+
}
237+
}
238+
239+
if (!matched) srcPos++;
240+
}
241+
242+
// Emit final literal-only sequence (no match, ML nibble ignored by spec)
243+
uint16_t finalLitLen = srcLen - litStart;
244+
dstPos = emitSequence(dst, dstPos, &src[litStart], finalLitLen, 0, 0);
245+
246+
// Must be even
247+
bool dec = false;
248+
if (dstPos & 1) {
249+
uint8_t i = dst[dstPos];
250+
dst[dstPos++] = i;
251+
dec = true;
252+
}
253+
254+
if (wordSwap) {
255+
uint16_t* p = (uint16_t*)dst;
256+
for (int i = 0; i < dstPos; i += 2) { *p = (*p >> 8) | (*p << 8); p++; }
257+
}
258+
259+
if (dec) dstPos--;
260+
261+
return dstPos;
130262
}

‎Pico/PaulaNET/rleCompression.h‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ _____ _ _ _ ___________
1616
#ifndef RLE_COMPRESSION
1717
#define RLE_COMPRESSION
1818

19-
#define RUN_BUFFER_SIZE 1600 // Max size of data that can be compressed and size of run buffer needed
19+
#define RUN_BUFFER_SIZE 2048 // Max size of data that can be compressed and size of run buffer needed or LZ4 compressor
2020

2121

2222
// Returns a guess as to the size of buffer needed to hold compressed data in a worse case situation
@@ -34,4 +34,7 @@ uint16_t rleDecompress(const uint8_t* src, uint16_t srclen, uint8_t* dst, uint16
3434
// Works out the space needed to decompress a buffer
3535
uint16_t rleCalcDecompressSize(const uint8_t* src, uint16_t srclen);
3636

37+
// Experimental LZ4 compressor
38+
size_t lz4Compress(const uint8_t* src, uint16_t srcLen, uint8_t* dst, bool wordSwap, uint16_t* hashBuffer);
39+
3740
#endif

0 commit comments

Comments
 (0)