In [18]:
text_splitter = RecursiveCharacterTextSplitter(
chunk_size=1000,
chunk_overlap=20,
separators=["\n\n", "\n", " ", ""]
)
documents = text_splitter.split_documents(documents=raw_documents)
documents[35:37]
Out[18]:
[Document(page_content='MkXeHYLS7Ex1Sq1NA03LRDc/qu4K5GJpMhpEtPAMCVc6ec3JuWU2I2JCW+2bp5+cDN9Mcg75Llm624rkbtaeBAzawSwSBIY/0NLcZnvjO4GNwo1RpO1WxjAjt3wR0vfYD5m45g2qsfYtDkO+Dm7YvinCxsX/ku3pw6BB//cyoO/PxVs1fubSzj9GFjVrIpwc0T67bi0dWJFnsviUGCOBxkLXOj8fSyGHLt3C8ecrkcXqaMjr6ywuZsQnEYKTimB6659X5c+/DTVtvZWlSx9k7k9a1yTNQRMHNjRXDXXrhw8G9knTvp7K60mLIC69NN/TtF4XJhPvIuXUBYj77ScbGQ0lqBpvn2C+Kbd+1N/2oT9+2pqqyQhrU4U6rt8vQLxMDrb8HA629BdVUVTu7cjH3fr8bp3Vtwfv8unN+/C9+pNegxfDyG3P4gug0e3WLp6rJC45CUTC6XMi72cPP2rVM/VDuoqL1pJmD8YOTpH4SCDOMeSuJsHpXWDVpPb1SUFKE4+4rVbIr4ugnt3gc3L3ijnr5ZD9bM14wCgArW3FAHx8yNFSFdjZmbrJTTTu5Jy7G1UJg0Y6pWmtx8OfvapO0XBIM0FbzBzI20BUO5NHWWa9y0DwqVCn3HTcYD767FMz8fwLWz/o3AqK7Q6ypxfOtGrJh9Oz6aeZPVWXmOIBbmunv72bX1Qn1qBwm2asP8zApWowbWrBUlvm5qr08jqhAL6j3q3+7CViaqzrCU9CHDt97zEbVXDG6sCI4xFhW7wnTw/T+swRtTrsGyByYh/eRhh5231MZCYeKMqdpFxWI63tpy9tIiftXVUjV7Q58YldIWDBXQlXGNm/bKN7QTxv/zKcz7dg+eWLcNw+76B5QaLS4k/4V37hyDEzs2O/wxxcyNu6/jdpivnW2xFdz0HTdZ+j5qwGDpezG4KbIR3IiF+A19KLD1nPS1h6VMmZuGMqhE7RWHpazwDYsEABRkXmqxDTTLiwuReng/IvoOgo', metadata={'source': 'test.ipynb', 'file_path': 'test.ipynb', 'file_name': 'test.ipynb', 'file_type': '.ipynb'}), Document(page_content='qxDTTLiwuReng/IvoOgoev9Rkd5/buxIYXjTue56alYPlDN2D0g3NQVVGGnIvnEHPVUAy5Y0addWcaQ1ptuNZjS5mbWrM3ikyFlN7WMjdSzU3NCsX21Nzodcbry8xN+yWTyRDWoy9ufPpVjLjnEXz13GxcSP4Lq/81Hfe8sQp9Rl/vsMeSgpsm1NvYUvv3WWsjwzL49gehKy9Dr5EJFq9LcaVkcc+r2moK8RvK3Fi+Xr2Dw1CUlVF3WErMoHawdW6IRAxurPAN7QTAuPZEaUGu1bVdmqO0IA8f3DsB+emp8AwIwuzPNsEvPNKiTXVVFb59eR4AIHb8DagsK8WZPVvx+7LFUptjW37BxUP7cOerH9o97dZm5sY0Y8o8cyNOawTqz9xUVZRLU7obDm5qtmAQF3xi5qZj8AuPxMzl3+KbF57AwV+/wbr5D+PRLzYhpGsvh5y/JrhpxjTwWmqv9Gtr+EipUmPsjCfrHPcONL5ubGVuxIynvcNSPsHhVoMbaRE/1txQB8VhKSuUao2URi7IuOTw8+/6cjnyTdNDS3Kz8cubz9Vpc+SPn5B36QI8/YNw6/Pv4oH31uGGp15G9MDBiLvxToydOQ8KpQpHEn/Ajs/es7sPUnDjW6vmJtK4rkNBRpr0hikOSSlUaqufhsWCYHH6qUwubzBQkYalKstrVifmNPAOQ6FS4bYX3kXXa0ahqqIMXz33KKqrqhxy7uZMA7dF7eZh8QGioSCkNilzYzO4aeywlGXA5hNi3Bi0bnBTAICzpajjYnBjg09IOADLvWQcQRAEJG/8BgAw/uFnIJPJcGzrL8hJtVy9dPeajwAY09xaTy/I5XIMv/thPLziZ9z24ntImD0fU//zOgBgyydvIT89za5+1MyWqpXmDgqF1ssHhupqqaDavJjY2hCdGNwc2/oLAGO9TUOFnFJBcUVFzb5SHgxuOhKFSoU7XvoAbj5+SD95WNqHqbnEBfzcHBjciCutiuxdbLKhgm', metadata={'source': 'test.ipynb', 'file_path': 'test.ipynb', 'file_name': 'test.ipynb', 'file_type': '.ipynb'})]
In [19]:
len(documents[35].page_content)
Out[19]:
1000