#include <vector>
#include <string_view>
#include <iostream>
auto tokenize(std::string_view string, std::string_view delimiters)
{
std::vector<std::string_view> substrings;
if (delimiters.size() == 0ul)
{
substrings.emplace_back(string);
return substrings;
}
auto start_pos = string.find_first_not_of(delimiters);
auto end_pos = start_pos;
auto max_length = string.length();
while (start_pos < max_length)
{
end_pos = std::min(max_length, string.find_first_of(delimiters, start_pos));
if (end_pos != start_pos)
{
substrings.emplace_back(&string[start_pos], end_pos - start_pos);
start_pos = string.find_first_not_of(delimiters, end_pos);
}
}
return substrings;
}
int main()
{
std::string_view test{ "The, quick! and brown fox. Jumped : over the lazy dog, or did he?" };
auto tokens = tokenize(test, " ,!.?:");
for (const auto token : tokens)
{
std::cout << token << "\n";
}
return 0;
}
