JIT: Added SVE GetFfr, SetFfr, LoadVectorFirstFaulting, GatherVectorFirstFaulting
#104502
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||||
|---|---|---|---|---|---|---|---|---|
|
|
@@ -9,10 +9,7 @@ public static unsafe partial class BoundedMemory | |||||||
| { | ||||||||
| private static UnixImplementation<T> AllocateWithoutDataPopulationUnix<T>(int elementCount, PoisonPagePlacement placement) where T : unmanaged | ||||||||
| { | ||||||||
| return new UnixImplementation<T>(elementCount, placement); | ||||||||
| } | ||||||||
|
|
||||||||
| private sealed class UnixImplementation<T> : BoundedMemory<T> where T : unmanaged | ||||||||
|
|
@@ -21,9 +18,9 @@ private sealed class UnixImplementation<T> : BoundedMemory<T> where T : unmanage | |||||||
| private readonly int _elementCount; | ||||||||
| private readonly BoundedMemoryManager _memoryManager; | ||||||||
|
|
||||||||
| public UnixImplementation(int elementCount, PoisonPagePlacement placement) | ||||||||
| { | ||||||||
| _handle = AllocHGlobalHandle.Allocate(checked(elementCount * (nint)sizeof(T)), placement); | ||||||||
| _elementCount = elementCount; | ||||||||
| _memoryManager = new BoundedMemoryManager(this); | ||||||||
| } | ||||||||
|
|
@@ -118,29 +115,77 @@ public override void Unpin() | |||||||
|
|
||||||||
| private sealed class AllocHGlobalHandle : SafeHandle | ||||||||
| { | ||||||||
| private IntPtr buffer; | ||||||||
| private ulong allocationSize; | ||||||||
|
|
||||||||
| // Called by P/Invoke when returning SafeHandles | ||||||||
| private AllocHGlobalHandle(IntPtr buffer, ulong allocationSize) | ||||||||
| : base(IntPtr.Zero, ownsHandle: true) | ||||||||
| { | ||||||||
| this.buffer = buffer; | ||||||||
| this.allocationSize = allocationSize; | ||||||||
| } | ||||||||
|
|
||||||||
| internal static AllocHGlobalHandle Allocate(nint byteLength, PoisonPagePlacement placement) | ||||||||
| { | ||||||||
|
|
||||||||
| // Allocate number of pages to incorporate required (byteLength bytes of) memory and an additional page to create a poison page. | ||||||||
| int pageSize = Environment.SystemPageSize; | ||||||||
| int allocationSize = (int)(((byteLength / pageSize) + ((byteLength % pageSize) == 0 ? 0 : 1) + 1) * pageSize); | ||||||||
| IntPtr buffer = mmap(IntPtr.Zero, (ulong)allocationSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); | ||||||||
|
|
||||||||
| if (buffer == IntPtr.Zero) | ||||||||
| { | ||||||||
| throw new InvalidOperationException($"Memory allocation failed with error {Marshal.GetLastPInvokeError()}."); | ||||||||
| } | ||||||||
|
|
||||||||
| // Depending on the PoisonPagePlacement requirement (before/after) initialise the baseAddress and poisonPageAddress to point to the location | ||||||||
| // in the buffer. Here the baseAddress points to the first valid allocation and poisonPageAddress points to the first invalid location. | ||||||||
| // For `PoisonPagePlacement.Before` the first page is made inaccessible using mprotect and baseAddress points to the start of the second page. | ||||||||
| // The allocation and protection is at the granularity of a page. Thus, `PoisonPagePlacement.Before` configuration has an additional accessible | ||||||||
| // memory at the end of the page (bytes equivalent to `pageSize - (byteLength % pageSize)`). | ||||||||
| // For `PoisonPagePlacement.After`, we adjust the baseAddress so that inaccessible memory is at the `byteLength` offset from the baseAddress. | ||||||||
| IntPtr baseAddress = buffer + pageSize; | ||||||||
| IntPtr poisonPageAddress = buffer; | ||||||||
| if (placement == PoisonPagePlacement.After) | ||||||||
| { | ||||||||
| baseAddress = buffer + (allocationSize - pageSize - byteLength); | ||||||||
| poisonPageAddress = buffer + (allocationSize - pageSize); | ||||||||
| } | ||||||||
|
|
||||||||
| // Protect the page before/after based on the poison page placement. | ||||||||
| if (mprotect(poisonPageAddress, (ulong) pageSize, PROT_NONE) == -1) | ||||||||
| { | ||||||||
| throw new InvalidOperationException($"Failed to mark page as a poison page using mprotect with error :{Marshal.GetLastPInvokeError()}."); | ||||||||
| } | ||||||||
|
|
||||||||
| AllocHGlobalHandle retVal = new AllocHGlobalHandle(buffer, (ulong)allocationSize); | ||||||||
| retVal.SetHandle(baseAddress); // this base address would be used as the start of Span that is used during unit testing. | ||||||||
| return retVal; | ||||||||
| } | ||||||||
|
|
||||||||
| public override bool IsInvalid => (handle == IntPtr.Zero); | ||||||||
|
|
||||||||
| protected override bool ReleaseHandle() | ||||||||
| { | ||||||||
| return munmap(buffer, allocationSize) == 0; | ||||||||
| } | ||||||||
|
|
||||||||
| // Defined in <sys/mman.h> | ||||||||
| const int MAP_PRIVATE = 0x2; | ||||||||
|
There was a problem hiding this comment.
Suggested change
There was a problem hiding this comment. Did you address all the feedback from Aaron - #104502 (review)? |
||||||||
| const int MAP_ANONYMOUS = 0x20; | ||||||||
| const int PROT_NONE = 0x0; | ||||||||
| const int PROT_READ = 0x1; | ||||||||
| const int PROT_WRITE = 0x2; | ||||||||
|
|
||||||||
| [DllImport("libc", SetLastError = true)] | ||||||||
| static extern IntPtr mmap(IntPtr address, ulong length, int prot, int flags, int fd, int offset); | ||||||||
|
|
||||||||
| [DllImport("libc", SetLastError = true)] | ||||||||
| static extern IntPtr munmap(IntPtr address, ulong length); | ||||||||
|
|
||||||||
| [DllImport("libc", SetLastError = true)] | ||||||||
| static extern int mprotect(IntPtr address, ulong length, int prot); | ||||||||
| } | ||||||||
| } | ||||||||
| } | ||||||||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@AaronRobinsonMSFT @tannergooding - can you review changes to this file as well. This adds page support for unix as well.