fix: block memory allocation overflow (#3639)

this fixes potential overflow bugs in pointer calculation by blocking
memory allocation above a certain size. the size limit is set at
`2**64`, which is the size of addressable memory on physical machines.

practically, for EVM use cases, we could limit at a much smaller number
(like `2**24`), but we want to allow for "exotic" targets which may
allow much more addressable memory.

vyper/codegen/memory_allocator.py

@@ -1,6 +1,6 @@
 from typing import List
 
-from vyper.exceptions import CompilerPanic
+from vyper.exceptions import CompilerPanic, MemoryAllocationException
 from vyper.utils import MemoryPositions
 
 
@@ -46,6 +46,8 @@ class MemoryAllocator:
 
     next_mem: int
 
+    _ALLOCATION_LIMIT: int = 2**64
+
     def __init__(self, start_position: int = MemoryPositions.RESERVED_MEMORY):
         """
         Initializer.
@@ -110,6 +112,14 @@ def _expand_memory(self, size: int) -> int:
         before_value = self.next_mem
         self.next_mem += size
         self.size_of_mem = max(self.size_of_mem, self.next_mem)
+
+        if self.size_of_mem >= self._ALLOCATION_LIMIT:
+            # this should not be caught
+            raise MemoryAllocationException(
+                f"Tried to allocate {self.size_of_mem} bytes! "
+                f"(limit is {self._ALLOCATION_LIMIT} (2**64) bytes)"
+            )
+
         return before_value
 
     def deallocate_memory(self, pos: int, size: int) -> None:

vyper/exceptions.py

@@ -269,6 +269,10 @@ class StorageLayoutException(VyperException):
     """Invalid slot for the storage layout overrides"""
 
 
+class MemoryAllocationException(VyperException):
+    """Tried to allocate too much memory"""
+
+
 class JSONError(Exception):
 
     """Invalid compiler input JSON."""