zstd.go 4.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152
  1. // Package zstd provides decompression of zstandard files.
  2. //
  3. // For advanced usage and examples, go to the README: https://github.com/klauspost/compress/tree/master/zstd#zstd
  4. package zstd
  5. import (
  6. "bytes"
  7. "encoding/binary"
  8. "errors"
  9. "log"
  10. "math"
  11. "math/bits"
  12. )
  13. // enable debug printing
  14. const debug = false
  15. // enable encoding debug printing
  16. const debugEncoder = debug
  17. // enable decoding debug printing
  18. const debugDecoder = debug
  19. // Enable extra assertions.
  20. const debugAsserts = debug || false
  21. // print sequence details
  22. const debugSequences = false
  23. // print detailed matching information
  24. const debugMatches = false
  25. // force encoder to use predefined tables.
  26. const forcePreDef = false
  27. // zstdMinMatch is the minimum zstd match length.
  28. const zstdMinMatch = 3
  29. // Reset the buffer offset when reaching this.
  30. const bufferReset = math.MaxInt32 - MaxWindowSize
  31. var (
  32. // ErrReservedBlockType is returned when a reserved block type is found.
  33. // Typically this indicates wrong or corrupted input.
  34. ErrReservedBlockType = errors.New("invalid input: reserved block type encountered")
  35. // ErrCompressedSizeTooBig is returned when a block is bigger than allowed.
  36. // Typically this indicates wrong or corrupted input.
  37. ErrCompressedSizeTooBig = errors.New("invalid input: compressed size too big")
  38. // ErrBlockTooSmall is returned when a block is too small to be decoded.
  39. // Typically returned on invalid input.
  40. ErrBlockTooSmall = errors.New("block too small")
  41. // ErrMagicMismatch is returned when a "magic" number isn't what is expected.
  42. // Typically this indicates wrong or corrupted input.
  43. ErrMagicMismatch = errors.New("invalid input: magic number mismatch")
  44. // ErrWindowSizeExceeded is returned when a reference exceeds the valid window size.
  45. // Typically this indicates wrong or corrupted input.
  46. ErrWindowSizeExceeded = errors.New("window size exceeded")
  47. // ErrWindowSizeTooSmall is returned when no window size is specified.
  48. // Typically this indicates wrong or corrupted input.
  49. ErrWindowSizeTooSmall = errors.New("invalid input: window size was too small")
  50. // ErrDecoderSizeExceeded is returned if decompressed size exceeds the configured limit.
  51. ErrDecoderSizeExceeded = errors.New("decompressed size exceeds configured limit")
  52. // ErrUnknownDictionary is returned if the dictionary ID is unknown.
  53. // For the time being dictionaries are not supported.
  54. ErrUnknownDictionary = errors.New("unknown dictionary")
  55. // ErrFrameSizeExceeded is returned if the stated frame size is exceeded.
  56. // This is only returned if SingleSegment is specified on the frame.
  57. ErrFrameSizeExceeded = errors.New("frame size exceeded")
  58. // ErrCRCMismatch is returned if CRC mismatches.
  59. ErrCRCMismatch = errors.New("CRC check failed")
  60. // ErrDecoderClosed will be returned if the Decoder was used after
  61. // Close has been called.
  62. ErrDecoderClosed = errors.New("decoder used after Close")
  63. // ErrDecoderNilInput is returned when a nil Reader was provided
  64. // and an operation other than Reset/DecodeAll/Close was attempted.
  65. ErrDecoderNilInput = errors.New("nil input provided as reader")
  66. )
  67. func println(a ...interface{}) {
  68. if debug || debugDecoder || debugEncoder {
  69. log.Println(a...)
  70. }
  71. }
  72. func printf(format string, a ...interface{}) {
  73. if debug || debugDecoder || debugEncoder {
  74. log.Printf(format, a...)
  75. }
  76. }
  77. // matchLenFast does matching, but will not match the last up to 7 bytes.
  78. func matchLenFast(a, b []byte) int {
  79. endI := len(a) & (math.MaxInt32 - 7)
  80. for i := 0; i < endI; i += 8 {
  81. if diff := load64(a, i) ^ load64(b, i); diff != 0 {
  82. return i + bits.TrailingZeros64(diff)>>3
  83. }
  84. }
  85. return endI
  86. }
  87. // matchLen returns the maximum length.
  88. // a must be the shortest of the two.
  89. // The function also returns whether all bytes matched.
  90. func matchLen(a, b []byte) int {
  91. b = b[:len(a)]
  92. for i := 0; i < len(a)-7; i += 8 {
  93. if diff := load64(a, i) ^ load64(b, i); diff != 0 {
  94. return i + (bits.TrailingZeros64(diff) >> 3)
  95. }
  96. }
  97. checked := (len(a) >> 3) << 3
  98. a = a[checked:]
  99. b = b[checked:]
  100. for i := range a {
  101. if a[i] != b[i] {
  102. return i + checked
  103. }
  104. }
  105. return len(a) + checked
  106. }
  107. func load3232(b []byte, i int32) uint32 {
  108. return binary.LittleEndian.Uint32(b[i:])
  109. }
  110. func load6432(b []byte, i int32) uint64 {
  111. return binary.LittleEndian.Uint64(b[i:])
  112. }
  113. func load64(b []byte, i int) uint64 {
  114. return binary.LittleEndian.Uint64(b[i:])
  115. }
  116. type byter interface {
  117. Bytes() []byte
  118. Len() int
  119. }
  120. var _ byter = &bytes.Buffer{}