Upgrade csbindgen to 1.9.3

sappho192 · Aug 26, 2024 · 87fb1ae · 87fb1ae
1 parent 404dd8b
commit 87fb1ae
Show file tree

Hide file tree

Showing 7 changed files with 16 additions and 16 deletions.
diff --git a/NATIVE_LIB_VERSION.txt b/NATIVE_LIB_VERSION.txt
@@ -1 +1 @@
-1.0.4
+1.0.5
diff --git a/dotnet/ConsoleExample/ConsoleExample.csproj b/dotnet/ConsoleExample/ConsoleExample.csproj
@@ -8,7 +8,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Tokenizers.DotNet.runtime.win" Version="0.7.0" />
+    <PackageReference Include="Tokenizers.DotNet.runtime.win" Version="1.0.5" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/dotnet/Tokenizers.DotNet/NativeMethods.cs b/dotnet/Tokenizers.DotNet/NativeMethods.cs
@@ -17,28 +17,28 @@ internal static unsafe partial class NativeMethods
 
 
         [DllImport(__DllName, EntryPoint = "alloc_u8_string", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-        public static extern ByteBuffer* alloc_u8_string();
+        internal static extern ByteBuffer* alloc_u8_string();
 
         [DllImport(__DllName, EntryPoint = "free_u8_string", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-        public static extern void free_u8_string(ByteBuffer* buffer);
+        internal static extern void free_u8_string(ByteBuffer* buffer);
 
         [DllImport(__DllName, EntryPoint = "csharp_to_rust_string", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-        public static extern void csharp_to_rust_string(ushort* utf16_str, int utf16_len);
+        internal static extern void csharp_to_rust_string(ushort* utf16_str, int utf16_len);
 
         [DllImport(__DllName, EntryPoint = "csharp_to_rust_u32_array", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-        public static extern void csharp_to_rust_u32_array(uint* buffer, int len);
+        internal static extern void csharp_to_rust_u32_array(uint* buffer, int len);
 
         [DllImport(__DllName, EntryPoint = "tokenizer_initialize", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-        public static extern ByteBuffer* tokenizer_initialize(ushort* utf16_path, int utf16_path_len);
+        internal static extern ByteBuffer* tokenizer_initialize(ushort* utf16_path, int utf16_path_len);
 
         [DllImport(__DllName, EntryPoint = "tokenizer_encode", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-        public static extern ByteBuffer* tokenizer_encode(ushort* _session_id, int _session_id_len, ushort* _text, int _text_len);
+        internal static extern ByteBuffer* tokenizer_encode(ushort* _session_id, int _session_id_len, ushort* _text, int _text_len);
 
         [DllImport(__DllName, EntryPoint = "tokenizer_decode", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-        public static extern ByteBuffer* tokenizer_decode(ushort* _session_id, int _session_id_len, uint* _token_ids, int _token_ids_len);
+        internal static extern ByteBuffer* tokenizer_decode(ushort* _session_id, int _session_id_len, uint* _token_ids, int _token_ids_len);
 
         [DllImport(__DllName, EntryPoint = "get_version", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-        public static extern ByteBuffer* get_version(ushort* _session_id, int _session_id_len);
+        internal static extern ByteBuffer* get_version(ushort* _session_id, int _session_id_len);
 
 
     }

diff --git a/dotnet/Tokenizers.DotNet/Tokenizers.DotNet.csproj b/dotnet/Tokenizers.DotNet/Tokenizers.DotNet.csproj
@@ -25,7 +25,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="csbindgen" Version="1.9.1">
+    <PackageReference Include="csbindgen" Version="1.9.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

diff --git a/nuget/Tokenizers.DotNet.nuspec b/nuget/Tokenizers.DotNet.nuspec
@@ -2,7 +2,7 @@
 <package >
   <metadata>
     <id>Tokenizers.DotNet</id>
-    <version>1.0.1</version>
+    <version>1.0.5</version>
     <title>Tokenizers.DotNet</title>
     <authors>sappho192</authors>
     <owners>sappho192</owners>

diff --git a/nuget/win-x64/Tokenizers.DotNet.runtime.win.nuspec b/nuget/win-x64/Tokenizers.DotNet.runtime.win.nuspec
@@ -2,7 +2,7 @@
 <package >
   <metadata>
     <id>Tokenizers.DotNet.runtime.win</id>
-    <version>1.0.1</version>
+    <version>1.0.5</version>
     <title>Tokenizers.DotNet.runtime.win</title>
     <authors>sappho192</authors>
     <owners>sappho192</owners>
@@ -11,7 +11,7 @@
     <!-- <icon>icon.png</icon> -->
     <projectUrl>https://github.com/sappho192/Tokenizers.DotNet</projectUrl>
     <repository type="git" url="https://github.com/sappho192/Tokenizers.DotNet.git" branch="$RepositoryBranch$" commit="$RepositoryCommit$" />
-    <description>Native(Rust) wrapper of HuggingFace Tokenizers library. Only for personal use.</description>
+    <description>Native(Rust) wrapper of HuggingFace Tokenizers library.</description>
     <copyright>Copyright (c) Taein Kim(sappho192)</copyright>
     <tags>tokenizers;rust;huggingface;tokenizer;gpt</tags>
   </metadata>

diff --git a/rust/Cargo.toml b/rust/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "hf_tokenizers"
-version = "1.0.1"
+version = "1.0.5"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -9,7 +9,7 @@ edition = "2021"
 crate-type = ["cdylib"]
 
 [build-dependencies]
-csbindgen = "1.9.1"
+csbindgen = "1.9.3"
 
 [dependencies]
 tokenizers = "0.20.0"