During the past week I managed to write the initial working version of the TrivialAugument. I only really managed to test it in training for 10 epochs (and only once so I don't know whether the output looks good because an easy image got chosen at random) but the outputs from the net already look much better than they looked before on 10th epoch.

So far TrivialAugument has 3 different operations: rotate, translate and scale. I also want to add a smooth deformation operation but I want it to use only operations from tinygrad so that we won't need to perform costly conversions from Pillow Image data type to Tensor and back. It would be great if in the future we'd be able to eliminate Pillow from the stack completely. Pillow's operations are not GPU accelerated and therefore they are somewhat slow. The biggest problem with Pillow though is that the conversion to numpy array and then to Tensor is incredibly slow.

class TrivialAugument:
  """
  This class is to be used in a separate script than the one that contains the training routine.
  During training this class will generate new transforms of the training dataset which will be
  saved in new .safetensors files which then will be ingested by the training routine.
  The whole thing is to run in a separate thread so that it doesn't slow down the training process.
  """
  def __init__(self, dataset: list[Tensor]):
    # Convert tensors to PIL images,
    self.images, self.labels = [Image.fromarray(make_8bit(x)) for x in dataset[0]], [Image.fromarray(make_8bit(x)) for x in dataset[1]]
    self.transformations = [self.rotate, self.zoom, self.translate]
   
  def rotate(self, s: float) -> Transform:
      print(f"rotate({s})")
      def f(img: Image.Image) -> Image.Image:
        angle = s * 45
        # Original dimensions
        w, h = img.size
        
        # Convert angle to radians for math functions
        angle_rad = math.radians(angle)
        
        # Compute the expanded bounding box size after rotation (for the canvas)
        cos_a = abs(math.cos(angle_rad))
        sin_a = abs(math.sin(angle_rad))
        new_w = int(math.ceil(w * cos_a + h * sin_a))
        new_h = int(math.ceil(w * sin_a + h * cos_a))
        
        # Perform the rotation with expansion (transparent fill for corners)
        rotated = img.rotate(angle, resample=Image.BICUBIC, expand=True)
        
        width_is_longer = w >= h
        side_long, side_short = (w, h) if width_is_longer else (h, w)
        
        sin_a_val, cos_a_val = abs(math.sin(angle_rad)), abs(math.cos(angle_rad))
        
        if side_short <= 2.0 * sin_a_val * cos_a_val * side_long or abs(sin_a_val - cos_a_val) < 1e-10:
            # Half-constrained: two crop corners touch the longer side
            x = 0.5 * side_short
            wr, hr = (x / sin_a_val, x / cos_a_val) if width_is_longer else (x / cos_a_val, x / sin_a_val)
        else:
            # Fully-constrained: crop touches all 4 sides
            cos_2a = cos_a_val * cos_a_val - sin_a_val * sin_a_val
            wr = (w * cos_a_val - h * sin_a_val) / cos_2a
            hr = (h * cos_a_val - w * sin_a_val) / cos_2a
        
        # Ensure positive dimensions and round to int
        wr = max(0, int(math.floor(wr)))
        hr = max(0, int(math.floor(hr)))
        
        # Crop from the center of the rotated image
        left = (new_w - wr) // 2
        top = (new_h - hr) // 2
        right = left + wr
        bottom = top + hr
        
        cropped = rotated.crop((left, top, right, bottom))
        
        return cropped
      return f
    
  def zoom(self, s: float) -> Transform:
    print(f"zoom({s})")
    def f(img: Image.Image) -> Image.Image:
      # TODO: Maybe add support for zooming out and reflect?
      crop = SIZE * s * 0.25
      border = crop // 2
      return img.crop((border, border, SIZE - border, SIZE - border))
    return f
    
  def translate(self, s: float) -> Transform:
    print(f"translate({s})")
    def f(img: Image.Image) -> Image.Image:
      size = img.size[0]          # since square → width = height
      max_shift = size * 0.25     # 25% of side length
  
      # Angle: 0° = right, 90° = down, 180° = left, 270° = up
      angle_deg = s * 360
      angle_rad = math.radians(angle_deg)
  
      # Displacement vector (positive = content moves in that direction)
      dx = math.cos(angle_rad) * max_shift   # x: positive = right
      dy = math.sin(angle_rad) * max_shift   # y: positive = down
  
      # Crop amounts (we crop opposite to movement direction)
      left   = max(0,  dx)     # crop left   when moving content right
      right  = max(0, -dx)     # crop right  when moving content left
      top    = max(0,  dy)     # crop top    when moving content down
      bottom = max(0, -dy)     # crop bottom when moving content up
  
      # Create crop box (all values are safe since max_shift = 0.25×size)
      crop_box = (
          int(left),           # left
          int(top),            # top
          int(size - right),   # right
          int(size - bottom)   # bottom
      )
      return img.crop(crop_box)
    return f
    
  # def elastic_deformation(self, img: Image.Image, s: float) -> Image.Image:
  #   pass
    
  def apply(self, img: Image.Image, transform: Transform) -> Tensor: return (Tensor(make_array(transform(img))) / 255).interpolate((SIZE, SIZE), "nearest-exact").expand(1, 1, -1, -1)
  
  def run_transform(self, image: Image.Image, label: Image.Image) -> tuple[Tensor, Tensor]:
    transform = choice(self.transformations)(random())
    return self.apply(image, transform), self.apply(label, transform)
    
  def augument(self) -> list[Tensor]:
    print("Performing TrivialAugument on the dataset...")
    images: list[Tensor] = []
    labels: list[Tensor] = []
    for i in range(len(self.images)):
      image, label = self.run_transform(self.images[i], self.labels[i])
      images.append(image)
      labels.append(label)
    return [images[0].stack(*images[1:]).realize(), labels[0].stack(*labels[1:]).realize()]
copy button

I decided to always keep two versions of a given image in the TrivialAugument object: the original Tensor loaded from file and the Pillow image loaded from array. Also every transformation function will take in both: the tensor and image. Since some transforms will take in tensors and some will take images, all of them will return tuples with one empty position depending on which datatype they operate on. After all transformations will be performed a function will run over all transformed images and convert the ones that need to be converted to tensors.